-----============= acceptance-small: sanity-pcc ============----- Wed Apr 30 23:34:06 EDT 2025 mgs: CentOS Linux release 7.9.2009 (Core) MGS_OS_VERSION_ID=7 MGS_OS_ID=centos MGS_OS_VERSION_CODE=117440512 MGS_OS_ID_LIKE=rhel fedora centos mds1: CentOS Linux release 7.9.2009 (Core) MDS1_OS_ID_LIKE=rhel fedora centos MDS1_OS_ID=centos MDS1_OS_VERSION_ID=7 MDS1_OS_VERSION_CODE=117440512 ost1: CentOS Linux release 7.9.2009 (Core) OST1_OS_VERSION_CODE=117440512 OST1_OS_VERSION_ID=7 OST1_OS_ID_LIKE=rhel fedora centos OST1_OS_ID=centos client: CentOS Linux release 7.9.2009 (Core) CLIENT_OS_ID=centos CLIENT_OS_ID_LIKE=rhel fedora centos CLIENT_OS_VERSION_ID=7 CLIENT_OS_VERSION_CODE=117440512 === sanity-pcc: start setup 23:34:15 (1746070455) === Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) oleg311-client.virtnet: executing check_config_client /mnt/lustre oleg311-client.virtnet: Checking config lustre mounted on /mnt/lustre Checking servers environments Checking clients oleg311-client.virtnet environments Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800aa083000.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800b5cb0000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800aa083000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b5cb0000.idle_timeout=debug disable quota as required oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all === sanity-pcc: finish setup 23:34:33 (1746070473) === running as uid/gid/euid/egid 500/500/500/500, groups: 500 [true] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/lustre/d0_runas_test/f7056] nobody:x:99: oleg311-server: ls: cannot access /home/green/git/lustre-release/lustre/tests/except/sanity-pcc.*ex: No such file or directory excepting tests: Killing existing copytools on oleg311-client.virtnet pkill: pidfile not valid Try `pkill --help' for more information. Set HSM on and start Set sanity-hsm HSM policy == sanity-pcc test 1a: Test manual lfs pcc attach with manual HSM restore ========================================================== 23:34:58 (1746070498) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0855686 s, 613 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 8d7493af-8f40-4512-be3e-07178b46e2c5 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=8d7493af-8f40-4512-be3e-07178b46e2c5 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1a.sanity-pcc/d1a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' using SAVE_PROJECT_SUPPORTED=0 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.052518 s, 146 MB/s Restore testing... Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1a.sanity-pcc/f1a.sanity-pcc Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1a.sanity-pcc is idle now PASS 1a (13s) == sanity-pcc test 1b: Test manual lfs pcc attach with restore on remote access ========================================================== 23:35:11 (1746070511) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0836169 s, 627 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: ff9f0ff8-59ff-47d3-94ad-5ad49f92cea5 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=ff9f0ff8-59ff-47d3-94ad-5ad49f92cea5 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1b.sanity-pcc/d1b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' using SAVE_PROJECT_SUPPORTED=0 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0512357 s, 149 MB/s Restore testing... Waiting 200s for 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1b.sanity-pcc/f1b.sanity-pcc Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1b.sanity-pcc is idle now PASS 1b (11s) == sanity-pcc test 1c: Test automated attach using Project ID with manual HSM restore ========================================================== 23:35:22 (1746070522) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg311-client.virtnet /mnt/lustre (opts:) Stopping client oleg311-client.virtnet /mnt/lustre opts: Stopping clients: oleg311-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg311-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg311-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg311-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg311-server' oleg311-server: oleg311-server.virtnet: executing load_modules_local oleg311-server: Loading modules from /home/green/git/lustre-release/lustre oleg311-server: detected 4 online CPUs by sysfs oleg311-server: MODOPTS_LIBCFS= oleg311-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800aa085000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012a545000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800aa085000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012a545000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0755061 s, 694 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 897c8e7a-2e38-46e2-bfe0-1874f52ff255 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=897c8e7a-2e38-46e2-bfe0-1874f52ff255 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1c.sanity-pcc/d1c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.048444 s, 158 MB/s Restore testing... Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1c.sanity-pcc/f1c.sanity-pcc Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1c.sanity-pcc is idle now PASS 1c (67s) == sanity-pcc test 1d: Test Project ID with remote access ========================================================== 23:36:29 (1746070589) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg311-client.virtnet /mnt/lustre (opts:) Stopping client oleg311-client.virtnet /mnt/lustre opts: Stopping clients: oleg311-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg311-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg311-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg311-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg311-server' oleg311-server: oleg311-server.virtnet: executing load_modules_local oleg311-server: Loading modules from /home/green/git/lustre-release/lustre oleg311-server: detected 4 online CPUs by sysfs oleg311-server: MODOPTS_LIBCFS= oleg311-server: Force libcfs to create 2 CPU partitions oleg311-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory oleg311-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b5cb3000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012abc8800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b5cb3000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012abc8800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100129 s, 524 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: e026182e-bf6e-4d47-b025-9aff0ea62104 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e026182e-bf6e-4d47-b025-9aff0ea62104 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1d.sanity-pcc/d1d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0474864 s, 161 MB/s Restore testing... Attach and detach testing Start to detach the /mnt/lustre/d1d.sanity-pcc/f1d.sanity-pcc Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1d.sanity-pcc is idle now PASS 1d (53s) == sanity-pcc test 1e: Test RW-PCC with non-root user ==== 23:37:22 (1746070642) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_1e/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_1e/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: projid={100} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00931758 s, 110 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00307731 s, 333 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00522504 s, 196 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] touch: cannot touch '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] [bs=1024] [count=1] dd: failed to open '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre PASS 1e (9s) == sanity-pcc test 1f: Test auto RW-PCC cache with non-root user ========================================================== 23:37:31 (1746070651) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg311-client.virtnet /mnt/lustre (opts:) Stopping client oleg311-client.virtnet /mnt/lustre opts: Stopping clients: oleg311-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg311-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg311-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg311-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg311-server' oleg311-server: oleg311-server.virtnet: executing load_modules_local oleg311-server: Loading modules from /home/green/git/lustre-release/lustre oleg311-server: detected 4 online CPUs by sysfs oleg311-server: MODOPTS_LIBCFS= oleg311-server: Force libcfs to create 2 CPU partitions oleg311-server: libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff88012f3b9000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012f3ba800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012f3b9000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012f3ba800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625069 s, 839 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: f90a81c3-4090-47ed-875f-a97f7332114d Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f90a81c3-4090-47ed-875f-a97f7332114d (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00532495 s, 192 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00100743 s, 1.0 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [bs=256] [count=1] 1+0 records in 1+0 records out 256 bytes (256 B) copied, 0.0027963 s, 91.5 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] touch: cannot touch '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1f.sanity-pcc is idle now PASS 1f (71s) == sanity-pcc test 1g: General permission test for RW-PCC ========================================================== 23:38:42 (1746070722) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707891 s, 741 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1g.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: bae64282-4012-44d1-8ba4-28da4a952559 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1g.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=bae64282-4012-44d1-8ba4-28da4a952559 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1g.sanity-pcc/d1g.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00707747 s, 145 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00299079 s, 342 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f1g.sanity-pcc] detach: cannot detach '/mnt/lustre/f1g.sanity-pcc' from PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f1g.sanity-pcc] Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00881755 s, 116 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1g.sanity-pcc is idle now PASS 1g (7s) == sanity-pcc test 2a: Test multi open when creating ===== 23:38:49 (1746070729) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg311-client.virtnet /mnt/lustre (opts:) Stopping client oleg311-client.virtnet /mnt/lustre opts: Stopping clients: oleg311-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg311-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg311-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg311-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg311-server' oleg311-server: oleg311-server.virtnet: executing load_modules_local oleg311-server: Loading modules from /home/green/git/lustre-release/lustre oleg311-server: detected 4 online CPUs by sysfs oleg311-server: MODOPTS_LIBCFS= oleg311-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b35f3800.idle_timeout=debug osc.lustre-OST0000-osc-ffff880136a58000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b35f3800.idle_timeout=debug osc.lustre-OST0001-osc-ffff880136a58000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658306 s, 796 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d2a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 48834d13-8fa6-4582-a212-9107f7d254d6 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f2a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=48834d13-8fa6-4582-a212-9107f7d254d6 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d2a.sanity-pcc/d2a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/d2a.sanity-pcc/multiop vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.24670 client oleg311-client.virtnet multiop_bg started multiop_pid=24680 ---------- 1 root root 14 Apr 30 23:39 /mnt/pcc.d2a.sanity-pcc/d2a.sanity-pcc/2343/0x200002341:0x2:0x0 Stopping multiop_pid=24680 (kill 24680 on oleg311-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d2a.sanity-pcc is idle now PASS 2a (71s) == sanity-pcc test 2b: Test multi remote open when creating ========================================================== 23:40:00 (1746070800) SKIP: sanity-pcc test_2b Need 2 or more clients, have 1 SKIP 2b (2s) == sanity-pcc test 2c: Test multi open on different mount points when creating ========================================================== 23:40:02 (1746070802) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg311-client.virtnet /mnt/lustre (opts:) Stopping client oleg311-client.virtnet /mnt/lustre opts: Stopping clients: oleg311-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg311-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg311-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg311-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory loading modules on: 'oleg311-server' oleg311-server: oleg311-server.virtnet: executing load_modules_local oleg311-server: Loading modules from /home/green/git/lustre-release/lustre oleg311-server: detected 4 online CPUs by sysfs oleg311-server: MODOPTS_LIBCFS= oleg311-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b6ceb000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012abce000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b6ceb000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012abce000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154495 s, 339 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d2c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: c3bf0d13-9627-4e16-9681-cb89c56f46a7 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f2c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=c3bf0d13-9627-4e16-9681-cb89c56f46a7 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d2c.sanity-pcc/d2c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/d2c.sanity-pcc/f2c.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.27866 client oleg311-client.virtnet multiop_bg started multiop_pid=27876 Stopping multiop_pid=27876 (kill 27876 on oleg311-client.virtnet) multiopen_dataCleanup PCC backend on /mnt/lustre /mnt/pcc.d2c.sanity-pcc is idle now PASS 2c (76s) == sanity-pcc test 3a: Repeat attach/detach operations === 23:41:18 (1746070878) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_3a/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00472539 s, 217 kB/s Start to RW-PCC attach/detach the file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Repeat to RW-PCC attach/detach the same file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Start to RO-PCC attach/detach the file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Repeat to RO-PCC attach/detach the same file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Cleanup PCC backend on /mnt/lustre PASS 3a (6s) == sanity-pcc test 3b: Repeat attach/detach operations on multiple clients ========================================================== 23:41:24 (1746070884) SKIP: sanity-pcc test_3b Need 3 or more clients, have 1 SKIP 3b (1s) == sanity-pcc test 4: Auto cache test for mmap =========== 23:41:26 (1746070886) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg311-client.virtnet /mnt/lustre (opts:) Stopping client oleg311-client.virtnet /mnt/lustre opts: Stopping clients: oleg311-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg311-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg311-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg311-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg311-server' oleg311-server: oleg311-server.virtnet: executing load_modules_local oleg311-server: Loading modules from /home/green/git/lustre-release/lustre oleg311-server: detected 4 online CPUs by sysfs oleg311-server: MODOPTS_LIBCFS= oleg311-server: Force libcfs to create 2 CPU partitions oleg311-server: libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b5cb3800.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012abca000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b5cb3800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012abca000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610575 s, 859 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d4.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 19b67be5-479b-488f-98d9-ba202d647ea0 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f4.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=19b67be5-479b-488f-98d9-ba202d647ea0 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d4.sanity-pcc/d4.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' mmap test1: basic mmap operation (PASS, 0.033715s) mmap test2: MAP_PRIVATE not write back (PASS, 0.027035s) mmap test3: concurrent mmap ops on two nodes (PASS, 2.0286s) mmap test4: c1 write to f1 from mmapped f2, c2 write to f1 from mmapped f1 (PASS, 4.1034s) mmap test5: read/write file to/from the buffer which mmapped to just this file (PASS, 0.020794s) mmap test6: check mmap write/read content on two nodes (PASS, 0.96918s) mmap test7: file i/o with an unmapped buffer (SKIPPED, 0s) mmap test8: SIGBUS for beyond file size (SKIPPED, 0s) mmap test9: SIGBUS for negative file offset (SKIPPED, 0s) mmap test10: multi-thread mmap access (PASS, 0.087136s) mmap test11: multi-thread shared mmap (PASS, 0.087461s) mmap test10: mtime not change for readonly mmap access (PASS, 1.0113s) ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b5cb3800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012abca000.lru_size=clear Cleanup PCC backend on /mnt/lustre /mnt/pcc.d4.sanity-pcc is idle now PASS 4 (58s) == sanity-pcc test 5: Mmap & cat a RW-PCC cached file ==== 23:42:23 (1746070943) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0724235 s, 724 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d5.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 1726407d-433c-4752-aee2-1db07c86f7d1 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f5.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=1726407d-433c-4752-aee2-1db07c86f7d1 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d5.sanity-pcc/d5.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d5.sanity-pcc is idle now PASS 5 (6s) == sanity-pcc test 6: Test mmap write on RW-PCC ========== 23:42:29 (1746070949) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725107 s, 723 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d6.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 840fe0f6-2ba7-4da6-ac10-ddca72d9d074 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f6.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=840fe0f6-2ba7-4da6-ac10-ddca72d9d074 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d6.sanity-pcc/d6.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d6.sanity-pcc is idle now PASS 6 (7s) == sanity-pcc test 7a: Fake file detached between fault() and page_mkwrite() for RW-PCC ========================================================== 23:42:36 (1746070956) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743393 s, 705 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d7a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 40748356-98c1-4025-885e-5b27659a695a Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f7a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=40748356-98c1-4025-885e-5b27659a695a (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d7a.sanity-pcc/d7a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1412 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d7a.sanity-pcc is idle now PASS 7a (5s) == sanity-pcc test 7b: Test the race with concurrent mkwrite and detach ========================================================== 23:42:41 (1746070961) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656817 s, 798 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d7b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 6741f8b0-294b-4f45-8a47-89efe5b1c88c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f7b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=6741f8b0-294b-4f45-8a47-89efe5b1c88c (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d7b.sanity-pcc/d7b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1413 fail_val=20 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d7b.sanity-pcc is idle now PASS 7b (46s) == sanity-pcc test 8: Test fake -ENOSPC tolerance for RW-PCC ========================================================== 23:43:27 (1746071007) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697851 s, 751 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d8.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 3205ea29-0a6b-4614-9c64-42df8af01dc8 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f8.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=3205ea29-0a6b-4614-9c64-42df8af01dc8 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d8.sanity-pcc/d8.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1411 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d8.sanity-pcc is idle now PASS 8 (5s) == sanity-pcc test 9: Test -ENOSPC tolerance on loop PCC device for RW-PCC ========================================================== 23:43:32 (1746071012) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111112 s, 472 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.9a: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 51711223-37ea-457a-a622-d8d419fec5fe Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f9.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=51711223-37ea-457a-a622-d8d419fec5fe (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.9a/d9.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.9a/d9.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 2.14615 s, 29.3 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.9a is idle now PASS 9 (6s) == sanity-pcc test 10a: Test RW-PCC with user quota on loop PCC device ========================================================== 23:43:38 (1746071018) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638794 s, 821 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a217d2f2-d803-4dbe-a077-7a87eb856d79 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a217d2f2-d803-4dbe-a077-7a87eb856d79 (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10a.sanity-pcc/d10a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10a.sanity-pcc/d10a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.132748 s, 118 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.134358 s, 117 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10a.sanity-pcc/uquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10a.sanity-pcc/uquotaB] lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x26:0x0]: Disk quota exceeded (122) lfs pcc pcc: cannot attach with ID: 2: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10a.sanity-pcc/uquotaB' to PCC with attach ID '2': Disk quota exceeded Test -EDQUOT error tolerance for RW-PCC running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d10a.sanity-pcc/uquotaA] [bs=1M] [count=30] 30+0 records in 30+0 records out 31457280 bytes (31 MB) copied, 1.04857 s, 30.0 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10a.sanity-pcc is idle now PASS 10a (6s) == sanity-pcc test 10b: Test RW-PCC with group quota on loop PCC device ========================================================== 23:43:44 (1746071024) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0843331 s, 622 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: f9616d22-fc7a-4865-b1a6-8202834dd9b2 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f9616d22-fc7a-4865-b1a6-8202834dd9b2 (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10b.sanity-pcc/d10b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10b.sanity-pcc/d10b.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.148002 s, 106 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.14724 s, 107 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10b.sanity-pcc/gquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10b.sanity-pcc/gquotaB] lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x2b:0x0]: Disk quota exceeded (122) lfs pcc pcc: cannot attach with ID: 2: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10b.sanity-pcc/gquotaB' to PCC with attach ID '2': Disk quota exceeded Test -EDQUOT error tolerance for RW-PCC running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d10b.sanity-pcc/gquotaA] [bs=1M] [count=30] 30+0 records in 30+0 records out 31457280 bytes (31 MB) copied, 0.60191 s, 52.3 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10b.sanity-pcc is idle now PASS 10b (5s) == sanity-pcc test 10c: Test RO-PCC with user quota on loop PCC device ========================================================== 23:43:49 (1746071029) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774443 s, 677 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 245a4733-f525-4f49-abcf-62655bc2856c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=245a4733-f525-4f49-abcf-62655bc2856c (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10c.sanity-pcc/d10c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10c.sanity-pcc/d10c.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.155809 s, 101 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.157735 s, 99.7 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10c.sanity-pcc/uquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10c.sanity-pcc/uquotaB] lfs pcc pcc: cannot attach the file to PCC with ID 2 failed: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10c.sanity-pcc/uquotaB' to PCC with attach ID '2': Disk quota exceeded Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10c.sanity-pcc is idle now PASS 10c (5s) == sanity-pcc test 10d: Test RO-PCC with group quota on loop PCC device ========================================================== 23:43:54 (1746071034) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0744401 s, 704 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 64368016-4cb2-4c66-b519-e833360cb6f9 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=64368016-4cb2-4c66-b519-e833360cb6f9 (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10d.sanity-pcc/d10d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10d.sanity-pcc/d10d.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.117813 s, 134 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.140875 s, 112 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10d.sanity-pcc/gquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10d.sanity-pcc/gquotaB] lfs pcc pcc: cannot attach the file to PCC with ID 2 failed: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10d.sanity-pcc/gquotaB' to PCC with attach ID '2': Disk quota exceeded Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10d.sanity-pcc is idle now PASS 10d (4s) == sanity-pcc test 10e: Tolerate -EDQUOT failure when auto PCC-RO attach with user quota ========================================================== 23:43:58 (1746071038) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661789 s, 792 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10e.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: d0a769a1-5743-4faf-aed0-15482f5c4777 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10e.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d0a769a1-5743-4faf-aed0-15482f5c4777 (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 4096K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 pcc: - pccpath: /mnt/pcc.d10e.sanity-pcc/d10e.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: uid={500} 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0345593 s, 60.7 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10e.sanity-pcc] [of=/dev/null] [bs=1M] [count=2] 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0114144 s, 184 MB/s /mnt/lustre/f10e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x34 lmm_fid: [0x2000032e1:0x34:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0744431 s, 70.4 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10e.sanity-pcc] [of=/dev/null] [bs=1M] [count=5] 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0244183 s, 215 MB/s file: /mnt/lustre/f10e.sanity-pcc, type: none /mnt/lustre/f10e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x34 lmm_fid: [0x2000032e1:0x34:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10e.sanity-pcc is idle now PASS 10e (4s) == sanity-pcc test 10f: Tolerate -EDQUOT failure when auto PCC-RO attach with group quota ========================================================== 23:44:02 (1746071042) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718475 s, 730 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 4ec5ae7f-5dd7-4faa-8181-32dcd8145dc8 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=4ec5ae7f-5dd7-4faa-8181-32dcd8145dc8 (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 4096K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 pcc: - pccpath: /mnt/pcc.d10f.sanity-pcc/d10f.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: gid={500} 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0234177 s, 89.6 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10f.sanity-pcc] [of=/dev/null] [bs=1M] [count=2] 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0147183 s, 142 MB/s /mnt/lustre/f10f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x36 lmm_fid: [0x2000032e1:0x36:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0758938 s, 69.1 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10f.sanity-pcc] [of=/dev/null] [bs=1M] [count=5] 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0238963 s, 219 MB/s file: /mnt/lustre/f10f.sanity-pcc, type: none /mnt/lustre/f10f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x36 lmm_fid: [0x2000032e1:0x36:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10f.sanity-pcc is idle now PASS 10f (5s) == sanity-pcc test 11: Test attach fault injection with simulated PCC file path ========================================================== 23:44:07 (1746071047) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717856 s, 730 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d11.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 9460b385-14f4-49b7-a2cd-7207b0f1d441 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f11.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=9460b385-14f4-49b7-a2cd-7207b0f1d441 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d11.sanity-pcc/d11.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Lustre file: /mnt/lustre/f11.sanity-pcc LPCC dir: /mnt/pcc.d11.sanity-pcc/d11.sanity-pcc/32d9 Lustre file: /mnt/lustre/f11.sanity-pcc LPCC dir: /mnt/pcc.d11.sanity-pcc/d11.sanity-pcc/32db lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x3a:0x0]: Permission denied (13) lfs pcc pcc: cannot attach with ID: 2: Permission denied (13) attach: cannot attach '/mnt/lustre/f11.sanity-pcc' to PCC with attach ID '2': Permission denied lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x3b:0x0]: Invalid argument (22) lfs pcc pcc: cannot attach with ID: 2: Invalid argument (22) attach: cannot attach '/mnt/lustre/f11.sanity-pcc' to PCC with attach ID '2': Invalid argument Cleanup PCC backend on /mnt/lustre /mnt/pcc.d11.sanity-pcc is idle now PASS 11 (5s) == sanity-pcc test 12: RW-PCC attach races with concurrent HSM remove ========================================================== 23:44:12 (1746071052) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_12/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1414 fail_val=20 /mnt/lustre/f12.sanity-pcc: (0x0000000d) released exists archived, archive_id:2 Cleanup PCC backend on /mnt/lustre PASS 12 (25s) == sanity-pcc test 13a: Test auto RW-PCC create caching for UID/GID rule ========================================================== 23:44:37 (1746071077) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_13a/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13a/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: uid={500} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00323995 s, 316 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.000330403 s, 3.1 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00759928 s, 135 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13a/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: gid={500} lfs mkdir: dirstripe error on '/mnt/lustre/d13a.sanity-pcc': stripe already set lfs setdirstripe: cannot create dir '/mnt/lustre/d13a.sanity-pcc': File exists running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00240153 s, 426 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.000590605 s, 1.7 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00318462 s, 322 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre PASS 13a (10s) == sanity-pcc test 13b: Test auto RW-PCC create caching for file name with wildcard ========================================================== 23:44:47 (1746071087) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_13b/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13b/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: fname={*.h5 suffix.* Mid*dle} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00470337 s, 218 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/suffix.doc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00856403 s, 120 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/MidPADdle] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00320408 s, 320 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/Midpad] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00657893 s, 156 kB/s Cleanup PCC backend on /mnt/lustre PASS 13b (4s) == sanity-pcc test 13c: Check auto RW-PCC create caching for UID/GID/ProjID/fname rule ========================================================== 23:44:51 (1746071091) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg311-client.virtnet /mnt/lustre (opts:) Stopping client oleg311-client.virtnet /mnt/lustre opts: Stopping clients: oleg311-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg311-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg311-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg311-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory loading modules on: 'oleg311-server' oleg311-server: oleg311-server.virtnet: executing load_modules_local oleg311-server: Loading modules from /home/green/git/lustre-release/lustre oleg311-server: detected 4 online CPUs by sysfs oleg311-server: MODOPTS_LIBCFS= oleg311-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b35f3000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012b661800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b35f3000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012b661800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102841 s, 510 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d13c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: e0a8f542-3d25-4fa8-85c9-3a0acd0b070c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f13c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e0a8f542-3d25-4fa8-85c9-3a0acd0b070c (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d13c.sanity-pcc/d13c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d13c.sanity-pcc/d13c.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100 200}&fname={*.h5},uid={500}&gid={500} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00874873 s, 117 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00612099 s, 167 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00892112 s, 115 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0127377 s, 80.4 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13c.sanity-pcc/ugidcache] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00675851 s, 152 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d13c.sanity-pcc is idle now PASS 13c (50s) == sanity-pcc test 14: Revocation of the layout lock should detach the file automatically ========================================================== 23:45:41 (1746071141) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_14/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b35f3000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012b661800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b35f3000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012b661800.lru_size=clear Cleanup PCC backend on /mnt/lustre PASS 14 (5s) == sanity-pcc test 15: Test auto attach at open when file is still valid cached ========================================================== 23:45:46 (1746071146) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738203 s, 710 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d15.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 114ccf9e-b428-4883-a499-e8948b27deb9 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f15.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=114ccf9e-b428-4883-a499-e8948b27deb9 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d15.sanity-pcc/d15.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Verify open attach for non-root user running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00923693 s, 111 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] file: /mnt/lustre/d15.sanity-pcc/f15.sanity-pcc, type: readwrite, PCC_file: /d15.sanity-pcc/3aa0/0x200003ab1:0x11:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b35f3000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012b661800.lru_size=clear running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] Verify auto attach at open for RW-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b35f3000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012b661800.lru_size=clear Waiting 200s for 'SUCCEED' Verify auto attach at open for RO-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b35f3000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012b661800.lru_size=clear Cleanup PCC backend on /mnt/lustre /mnt/pcc.d15.sanity-pcc is idle now PASS 15 (7s) == sanity-pcc test 16: Test detach with different options ========================================================== 23:45:53 (1746071153) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797824 s, 657 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d16.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 181b3ead-edbb-499d-a832-634e9a6bfd00 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f16.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=181b3ead-edbb-499d-a832-634e9a6bfd00 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d16.sanity-pcc/d16.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Test detach for RW-PCC Test for reusing valid PCC cache Test for the default detach Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Test detach for RO-PCC detach_dataCleanup PCC backend on /mnt/lustre /mnt/pcc.d16.sanity-pcc is idle now PASS 16 (9s) == sanity-pcc test 17: Test auto attach for layout refresh ========================================================== 23:46:02 (1746071162) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777204 s, 675 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d17.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 87dd3702-b6bb-4add-a529-915cce37fb47 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f17.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=87dd3702-b6bb-4add-a529-915cce37fb47 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d17.sanity-pcc/d17.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d17.sanity-pcc/d17.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 94 autocache: projid={100} Verify auto attach during IO for truncate Verify auto attach during IO for read/write multiop /mnt/lustre/f17.sanity-pcc vO_r TMPPIPE=/tmp/multiop_open_wait_pipe.19318 client oleg311-client.virtnet multiop_bg started multiop_pid=19328 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b35f3000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012b661800.lru_size=clear Stopping multiop_pid=19328 (kill 19328 on oleg311-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d17.sanity-pcc is idle now PASS 17 (13s) == sanity-pcc test 18: Verify size correctness after re-attach the file ========================================================== 23:46:15 (1746071175) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069259 s, 757 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d18.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: b43facb1-2ce0-4ce0-a5ff-da1888492ce6 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f18.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b43facb1-2ce0-4ce0-a5ff-da1888492ce6 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d18.sanity-pcc/d18.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d18.sanity-pcc/d18.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.0971476 s, 43.2 MB/s file: /mnt/lustre/f18.sanity-pcc, type: readwrite, PCC_file: /d18.sanity-pcc/3aac/0x200003ab1:0x1d:0x0, open_count: 0, flags: 0 file: /mnt/lustre/f18.sanity-pcc, type: none /mnt/lustre/f18.sanity-pcc has size 4194304 OK 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00297396 s, 344 kB/s /mnt/lustre2/f18.sanity-pcc has size 1049600 OK file: /mnt/lustre/f18.sanity-pcc, type: none Cleanup PCC backend on /mnt/lustre /mnt/pcc.d18.sanity-pcc is idle now PASS 18 (7s) == sanity-pcc test 19: Verify the file re-attach works as expected ========================================================== 23:46:22 (1746071182) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0937081 s, 559 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d19.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 33a002ae-e682-4ee2-b597-46486627b53f Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f19.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=33a002ae-e682-4ee2-b597-46486627b53f (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d19.sanity-pcc/d19.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d19.sanity-pcc is idle now PASS 19 (6s) == sanity-pcc test 20: Auto attach works after the inode was once evicted from cache ========================================================== 23:46:28 (1746071188) 120+0 records in 120+0 records out 125829120 bytes (126 MB) copied, 0.215313 s, 584 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d20.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/122880 done Creating filesystem with 122880 1k blocks and 30720 inodes Filesystem UUID: bf4420ea-99a8-4cc4-9ed3-aacf8bc2194d Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/15 done Writing inode tables: 0/15 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/15 done /tmp/f20.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=bf4420ea-99a8-4cc4-9ed3-aacf8bc2194d (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d20.sanity-pcc/d20.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d20.sanity-pcc is idle now PASS 20 (7s) == sanity-pcc test 21a: PCC-RO storing as a plain HSM mirror component for plain layout ========================================================== 23:46:35 (1746071195) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0992592 s, 528 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 1b0e5e0b-c36b-4935-bfea-9b09391d087d Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=1b0e5e0b-c36b-4935-bfea-9b09391d087d (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21a.sanity-pcc/d21a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Plain layout info before PCC-RO attach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 267 0x10b 0x240000400 FLR layout info after PCC-RO attach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after RO-PCC attach /mnt/lustre/f21a.sanity-pcc again: /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after RO-PCC detach '/mnt/lustre/f21a.sanity-pcc' again: /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21a.sanity-pcc is idle now PASS 21a (5s) == sanity-pcc test 21b: PCC-RO stroing as a plain HSM mirror component for FLR layouts ========================================================== 23:46:40 (1746071200) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732841 s, 715 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 4ab76a34-7954-417a-85fe-7a6a1f719c9d Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=4ab76a34-7954-417a-85fe-7a6a1f719c9d (extents) (64bit) (large files) (huge files) FLR layout before PCC-RO attach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: ro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout after PCC-RO attach '/mnt/lustre/f21b.sanity-pcc' again: /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21b.sanity-pcc is idle now PASS 21b (4s) == sanity-pcc test 21c: Verify HSM release works storing PCC-RO as HSM mirror component ========================================================== 23:46:45 (1746071205) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.082151 s, 638 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 56f03cfc-3a00-47d2-97d4-51d9d3d177bf Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=56f03cfc-3a00-47d2-97d4-51d9d3d177bf (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21c.sanity-pcc/d21c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' /mnt/lustre/f21c.sanity-pcc: (0x00000009) exists archived, archive_id:2 /mnt/lustre/f21c.sanity-pcc lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 0 lmm_stripe_offset: 0 lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' /mnt/lustre/f21c.sanity-pcc: (0x0000000d) released exists archived, archive_id:2 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21c.sanity-pcc is idle now PASS 21c (8s) == sanity-pcc test 21d: Write should invalidate PCC-RO caching ========================================================== 23:46:52 (1746071212) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703415 s, 745 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: d67ebcc2-6081-4c66-89cc-e6a3c3d4d7cf Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d67ebcc2-6081-4c66-89cc-e6a3c3d4d7cf (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21d.sanity-pcc/d21d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21d.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 270 0x10e 0x280000400 PCC-RO attach '/mnt/lustre/f21d.sanity-pcc': /mnt/lustre/f21d.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2d lmm_fid: [0x200003ab1:0x2d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10e:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 8594128896 lhm_archive_ver: 9663677440 lhm_archive_uuid: '' Write invalidated PCC-RO cache: /mnt/lustre/f21d.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2d lmm_fid: [0x200003ab1:0x2d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10e:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 4294967296 lhm_archive_ver: 4493216720 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21d.sanity-pcc is idle now PASS 21d (5s) == sanity-pcc test 21e: Truncate should invalidate PCC-RO caching ========================================================== 23:46:57 (1746071217) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0861912 s, 608 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21e.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 96c4e69e-801c-44bd-b170-e05c47e30ccf Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21e.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=96c4e69e-801c-44bd-b170-e05c47e30ccf (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21e.sanity-pcc/d21e.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21e.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 271 0x10f 0x240000400 PCC-RO attach '/mnt/lustre/f21e.sanity-pcc': /mnt/lustre/f21e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2f lmm_fid: [0x200003ab1:0x2f:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 8594128896 lhm_archive_ver: 9663677440 lhm_archive_uuid: '' Trucate invalidate PCC-RO file '/mnt/lustre/f21e.sanity-pcc': /mnt/lustre/f21e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2f lmm_fid: [0x200003ab1:0x2f:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21e.sanity-pcc is idle now PASS 21e (5s) == sanity-pcc test 21f: mmap write should invalidate PCC-RO caching ========================================================== 23:47:02 (1746071222) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668229 s, 785 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 6782d51e-749b-4087-a901-10be8b221a27 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=6782d51e-749b-4087-a901-10be8b221a27 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21f.sanity-pcc/d21f.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21f.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 271 0x10f 0x280000400 PCC-RO attach '/mnt/lustre/f21f.sanity-pcc': /mnt/lustre/f21f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x30 lmm_fid: [0x200003ab1:0x30:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 8594128896 lhm_archive_ver: 9663677440 lhm_archive_uuid: '' Mmap write invalidate PCC-RO caching: /mnt/lustre/f21f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x30 lmm_fid: [0x200003ab1:0x30:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21f.sanity-pcc is idle now PASS 21f (4s) == sanity-pcc test 21g: PCC-RO for file under FLR write pending state ========================================================== 23:47:06 (1746071226) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623902 s, 840 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21g.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 5fd23a7c-dede-4fca-a7b2-f6582a70012b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21g.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5fd23a7c-dede-4fca-a7b2-f6582a70012b (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21g.sanity-pcc/d21g.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' FLR layout before PCC-RO attach '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: wp lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x110:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x110:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x111:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x111:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: wp,pccro lcm_layout_gen: 4 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x110:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x110:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x111:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x111:0x0] } - lcme_id: 196612 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after Write invalidate '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: wp lcm_layout_gen: 5 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x110:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x110:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x111:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x111:0x0] } - lcme_id: 196612 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21g.sanity-pcc is idle now PASS 21g (5s) == sanity-pcc test 21h: Extend mirror once file was PCC-RO cached ========================================================== 23:47:11 (1746071231) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0723948 s, 724 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21h.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 4b9e0f0a-f381-486f-a1e6-a7ce25f612fb Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21h.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=4b9e0f0a-f381-486f-a1e6-a7ce25f612fb (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21h.sanity-pcc/d21h.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' FLR layout before PCC-RO attach '/mnt/lustre/f21h.sanity-pcc': /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: ro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21h.sanity-pcc': /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout after extend a mirror: /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 536 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 4 lcm_entry_count: 4 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 224 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 304 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 384 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' - lcme_id: 262145 lcme_mirror_id: 4 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 456 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 8388608 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x114:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x114:0x0] } file: /mnt/lustre/f21h.sanity-pcc, type: none FLR layout after PCC-RO attach '/mnt/lustre/f21h.sanity-pcc' again: /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 536 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 4 lcm_entry_count: 4 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 224 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 304 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 384 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' - lcme_id: 262145 lcme_mirror_id: 4 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 456 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 8388608 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x114:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x114:0x0] } Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21h.sanity-pcc is idle now PASS 21h (5s) == sanity-pcc test 21i: HSM release increase layout gen, should invalidate PCC-RO cache ========================================================== 23:47:16 (1746071236) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0872044 s, 601 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21i.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 33d164aa-f444-4c13-a82c-bc1622641a4f Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21i.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=33d164aa-f444-4c13-a82c-bc1622641a4f (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21i.sanity-pcc/d21i.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21i.sanity-pcc: (0x00000009) exists archived, archive_id:2 Layout after HSM release /mnt/lustre/f21i.sanity-pcc: /mnt/lustre/f21i.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x37 lmm_fid: [0x200003ab1:0x37:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 2 lmm_stripe_offset: 0 - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' PCC state /mnt/lustre/f21i.sanity-pcc: file: /mnt/lustre/f21i.sanity-pcc, type: none Layout after PCC-RO attach /mnt/lustre/f21i.sanity-pcc again: /mnt/lustre/f21i.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x37 lmm_fid: [0x200003ab1:0x37:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 0 obdidx objid objid group 0 278 0x116 0x240000400 PCC state: file: /mnt/lustre/f21i.sanity-pcc, type: readonly, PCC_file: /d21i.sanity-pcc/3a86/0x200003ab1:0x37:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21i.sanity-pcc is idle now PASS 21i (6s) == sanity-pcc test 21j: PCC-RO for encrypted file ======== 23:47:22 (1746071242) SKIP: sanity-pcc test_21j client encryption not supported SKIP 21j (1s) == sanity-pcc test 22: Test RO-PCC attach for the HSM released file ========================================================== 23:47:23 (1746071243) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110149 s, 476 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d22.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: d8173409-aee1-4843-a69a-2ee6a7817f58 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f22.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d8173409-aee1-4843-a69a-2ee6a7817f58 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d22.sanity-pcc/d22.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Layout after PCC-RO attach /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 2 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x117:0x0] } - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after HSM release /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 2 lmm_stripe_offset: 16 - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after PCC-RO attach /mnt/lustre/f22.sanity-pcc again: /mnt/lustre/f22.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 1 obdidx objid objid group 1 279 0x117 0x280000400 Layout after PCC-RO detach /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 1 obdidx objid objid group 1 279 0x117 0x280000400 Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d22.sanity-pcc is idle now PASS 22 (10s) == sanity-pcc test 23: Test write/truncate/mmap-write invalidating RO-PCC caching ========================================================== 23:47:33 (1746071253) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781599 s, 671 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d23.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 7741fe35-1d38-48f6-adef-7db16ea5305a Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f23.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=7741fe35-1d38-48f6-adef-7db16ea5305a (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d23.sanity-pcc/d23.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Write should invalidate the RO-PCC cache: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' PCC-RO attach '/mnt/lustre/f23.sanity-pcc' again: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' Truncate invalidate the RO-PCC cache: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 6 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' Finish trucate operation Mmap write invalidates RO-PCC caching /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 6 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' PCC-RO attach '/mnt/lustre/f23.sanity-pcc' again: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 7 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' Mmap write /mnt/lustre/f23.sanity-pcc via multiop /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 8 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 0 lhm_archive_ver: 4493216720 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d23.sanity-pcc is idle now PASS 23 (6s) == sanity-pcc test 24a: Test RO-PCC with non-root user === 23:47:39 (1746071259) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810736 s, 647 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d24a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 69333746-cd0f-4b1f-b6a1-4ac5930fb320 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f24a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=69333746-cd0f-4b1f-b6a1-4ac5930fb320 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: a0 autocache: projid={100} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00931489 s, 110 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [-i] [2] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00273831 s, 374 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0] touch: cannot touch '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [-i] [2] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] Cleanup PCC backend on /mnt/lustre /mnt/pcc.d24a.sanity-pcc is idle now PASS 24a (5s) == sanity-pcc test 24b: General permission test for RO-PCC ========================================================== 23:47:44 (1746071264) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0834466 s, 628 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d24b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 3afcd5ba-f6c4-4a89-925f-057003bdffa4 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f24b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=3afcd5ba-f6c4-4a89-925f-057003bdffa4 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d24b.sanity-pcc/d24b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0067394 s, 152 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00312212 s, 328 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] detach: cannot detach '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc' from PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00798708 s, 128 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d24b.sanity-pcc is idle now PASS 24b (5s) == sanity-pcc test 25: Tolerate fake read failure for RO-PCC ========================================================== 23:47:49 (1746071269) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077119 s, 680 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d25.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: d6b9da9c-5dd6-4857-9a9a-6db088cf3362 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f25.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d6b9da9c-5dd6-4857-9a9a-6db088cf3362 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d25.sanity-pcc/d25.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1411 fail_loc=0 fail_loc=0x1411 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d25.sanity-pcc is idle now PASS 25 (5s) == sanity-pcc test 26: Repeat the attach/detach when the file has multiple openers ========================================================== 23:47:54 (1746071274) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114198 s, 459 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d26.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: c73ab902-96f1-433c-b522-15c066f3ee8f Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f26.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=c73ab902-96f1-433c-b522-15c066f3ee8f (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d26.sanity-pcc/d26.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2591 client oleg311-client.virtnet multiop_bg started multiop_pid=2601 Stopping multiop_pid=2601 (kill 2601 on oleg311-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2706 client oleg311-client.virtnet multiop_bg started multiop_pid=2716 Stopping multiop_pid=2716 (kill 2716 on oleg311-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2821 client oleg311-client.virtnet multiop_bg started multiop_pid=2831 Stopping multiop_pid=2831 (kill 2831 on oleg311-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2936 client oleg311-client.virtnet multiop_bg started multiop_pid=2946 Waiting 200s for 'SUCCEED' Stopping multiop_pid=2946 (kill 2946 on oleg311-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.3086 client oleg311-client.virtnet multiop_bg started multiop_pid=3096 Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Stopping multiop_pid=3096 (kill 3096 on oleg311-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d26.sanity-pcc is idle now PASS 26 (27s) == sanity-pcc test 27: Auto attach at open when the file has multiple openers ========================================================== 23:48:21 (1746071301) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107726 s, 487 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d27.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 3fa96fac-9553-4dbe-bcdf-f2d0d3c85077 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f27.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=3fa96fac-9553-4dbe-bcdf-f2d0d3c85077 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d27.sanity-pcc/d27.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4003 client oleg311-client.virtnet multiop_bg started multiop_pid=4013 Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Stopping multiop_pid=4013 (kill 4013 on oleg311-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4146 client oleg311-client.virtnet multiop_bg started multiop_pid=4156 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b35f3000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012b661800.lru_size=clear Waiting 200s for 'SUCCEED' Stopping multiop_pid=4156 (kill 4156 on oleg311-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4290 client oleg311-client.virtnet multiop_bg started multiop_pid=4300 Stopping multiop_pid=4300 (kill 4300 on oleg311-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4409 client oleg311-client.virtnet multiop_bg started multiop_pid=4419 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b35f3000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012b661800.lru_size=clear Stopping multiop_pid=4419 (kill 4419 on oleg311-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d27.sanity-pcc is idle now PASS 27 (23s) == sanity-pcc test 28: RW-PCC attach should fail when the file has cluster-wide openers ========================================================== 23:48:44 (1746071324) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0750448 s, 699 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d28.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 95ae324b-32c3-4a43-ad59-229726dfe984 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f28.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=95ae324b-32c3-4a43-ad59-229726dfe984 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d28.sanity-pcc/d28.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f28.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.5276 client oleg311-client.virtnet multiop_bg started multiop_pid=5286 lfs pcc pcc: cannot get WRITE lease, ext 0 for [0x200003ab1:0x5f:0x0]: Device or resource busy (16) lfs pcc pcc: cannot get lease: Device or resource busy (16) attach: cannot attach '/mnt/lustre/f28.sanity-pcc' to PCC with attach ID '2': Device or resource busy Stopping multiop_pid=5286 (kill 5286 on oleg311-client.virtnet) multiop /mnt/lustre2/f28.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.7056 lfs pcc pcc: cannot get WRITE lease, ext 0 for [0x200003ab1:0x5f:0x0]: Device or resource busy (16) lfs pcc pcc: cannot get lease: Device or resource busy (16) attach: cannot attach '/mnt/lustre/f28.sanity-pcc' to PCC with attach ID '2': Device or resource busy Cleanup PCC backend on /mnt/lustre /mnt/pcc.d28.sanity-pcc is idle now PASS 28 (8s) == sanity-pcc test 29a: Auto readonly caching on RO-PCC backend for O_RDONLY open ========================================================== 23:48:52 (1746071332) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg311-client.virtnet /mnt/lustre (opts:) Stopping client oleg311-client.virtnet /mnt/lustre opts: Stopping clients: oleg311-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg311-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg311-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg311-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg311-server' oleg311-server: oleg311-server.virtnet: executing load_modules_local oleg311-server: Loading modules from /home/green/git/lustre-release/lustre oleg311-server: detected 4 online CPUs by sysfs oleg311-server: MODOPTS_LIBCFS= oleg311-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b6fed800.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012a457800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b6fed800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012a457800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592915 s, 884 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d29a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 346ed3e0-4084-49c0-b9c6-f34f4340e423 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f29a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=346ed3e0-4084-49c0-b9c6-f34f4340e423 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d29a.sanity-pcc/d29a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d29a.sanity-pcc/d29a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 100 P /mnt/lustre/d29a.sanity-pcc/f29a.sanity-pcc Cleanup PCC backend on /mnt/lustre /mnt/pcc.d29a.sanity-pcc is idle now PASS 29a (40s) == sanity-pcc test 29b: Auto PCC-RO attach in atomic_open ========================================================== 23:49:32 (1746071372) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0709229 s, 739 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d29b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 00d0d85f-3d0c-4ee4-97cb-1b630a0a02c7 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f29b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=00d0d85f-3d0c-4ee4-97cb-1b630a0a02c7 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d29b.sanity-pcc/d29b.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: fname={*.dat} 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.00936583 s, 437 kB/s 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.00603737 s, 678 kB/s file: /mnt/lustre/myfile.dat, type: readonly, PCC_file: /d29b.sanity-pcc/4282/0x200004281:0x3:0x0, open_count: 0, flags: 0 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.00364314 s, 1.1 MB/s vm.drop_caches = 3 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.00761399 s, 538 kB/s file: /mnt/lustre/myfile.dat, type: readonly, PCC_file: /d29b.sanity-pcc/4282/0x200004281:0x3:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d29b.sanity-pcc is idle now PASS 29b (4s) == sanity-pcc test 30: Test lctl pcc del command ========= 23:49:36 (1746071376) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806675 s, 650 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d30.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 10ffe063-36ed-4718-8e4b-d36580631f41 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f30.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=10ffe063-36ed-4718-8e4b-d36580631f41 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d30.sanity-pcc/d30.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: 3e autocache: projid={100} check file 0:'4287' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 0:'428b' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 0:'4289' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 1:'0x200004281:0x6:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4287' Handle the file: 0x200004281:0x6:0x0 Detach and remove the PCC cached file: 0x200004281:0x6:0x0 check file 1:'0x200004281:0xa:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/428b' Handle the file: 0x200004281:0xa:0x0 Detach and remove the PCC cached file: 0x200004281:0xa:0x0 check file 1:'0x200004281:0x8:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4289' Handle the file: 0x200004281:0x8:0x0 Remove non-cached file: /mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4289/0x200004281:0x8:0x0 flags: 1 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d30.sanity-pcc is idle now PASS 30 (6s) == sanity-pcc test 31: Test lctl pcc del command with --keep option ========================================================== 23:49:42 (1746071382) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729574 s, 719 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d31.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 37d6ae10-ee63-4cd8-9f10-c1dcef947c31 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f31.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=37d6ae10-ee63-4cd8-9f10-c1dcef947c31 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d31.sanity-pcc/d31.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d31.sanity-pcc/d31.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: 30 autocache: projid={100} Cleanup PCC backend on /mnt/lustre /mnt/pcc.d31.sanity-pcc is idle now PASS 31 (5s) == sanity-pcc test 32: Test for RO-PCC when PCC copy is deleted ========================================================== 23:49:47 (1746071387) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0713785 s, 735 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d32.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 6a104637-260c-4e71-a624-d2648645de39 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f32.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=6a104637-260c-4e71-a624-d2648645de39 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d32.sanity-pcc/d32.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f32.sanity-pcc vo_rc TMPPIPE=/tmp/multiop_open_wait_pipe.11538 client oleg311-client.virtnet multiop_bg started multiop_pid=11548 Stopping multiop_pid=11548 (kill 11548 on oleg311-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d32.sanity-pcc is idle now PASS 32 (10s) == sanity-pcc test 33: Cache rule with comparator (>, =, <) for file size ========================================================== 23:49:57 (1746071397) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0676735 s, 775 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d33.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 37e5fea6-aef1-4ca0-9ef8-fc8a579ea8c3 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f33.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=37e5fea6-aef1-4ca0-9ef8-fc8a579ea8c3 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d33.sanity-pcc/d33.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: fname={*.doc}&size<{1M} file: /mnt/lustre/myfile.doc, type: none file: /mnt/lustre/myfile.doc, type: none Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: fname={*.doc}&size<{5M}&size>{3M} Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: fname={*.doc}&size={5M 3M} Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre /mnt/pcc.d33.sanity-pcc is idle now PASS 33 (5s) == sanity-pcc test 34: Cache rule with comparator (>, <) for Project ID range ========================================================== 23:50:02 (1746071402) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg311-client.virtnet /mnt/lustre (opts:) Stopping client oleg311-client.virtnet /mnt/lustre opts: Stopping clients: oleg311-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg311-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg311-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg311-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg311-server' oleg311-server: oleg311-server.virtnet: executing load_modules_local oleg311-server: Loading modules from /home/green/git/lustre-release/lustre oleg311-server: detected 4 online CPUs by sysfs oleg311-server: MODOPTS_LIBCFS= oleg311-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a6129800.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800a862b000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a6129800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a862b000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608607 s, 861 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d34.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: ddb4f0dd-2c6a-4b74-a311-677281dfc4db Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f34.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=ddb4f0dd-2c6a-4b74-a311-677281dfc4db (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d34.sanity-pcc/d34.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid>{100} 99 - /mnt/lustre/f34.sanity-pcc 101 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid<{100} 102 - /mnt/lustre/f34.sanity-pcc 99 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid<{120}&projid>{110} 105 - /mnt/lustre/f34.sanity-pcc 121 - /mnt/lustre/f34.sanity-pcc 115 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre /mnt/pcc.d34.sanity-pcc is idle now PASS 34 (46s) == sanity-pcc test 35: mmap fault test =================== 23:50:48 (1746071448) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635385 s, 825 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d35.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: f99c9b82-64be-42fb-ad7c-495b683ffd3c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f35.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f99c9b82-64be-42fb-ad7c-495b683ffd3c (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d35.sanity-pcc/d35.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d35.sanity-pcc is idle now PASS 35 (3s) == sanity-pcc test 36a: Stale RW-PCC copy should be deleted after remove the PCC backend ========================================================== 23:50:51 (1746071451) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749444 s, 700 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d36a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: c0179ef9-1140-4d88-90a1-be45c97c4933 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f36a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=c0179ef9-1140-4d88-90a1-be45c97c4933 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' check file 0:'4a52' under directory '/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc' check file 1:'0x200004a51:0x3:0x0' under directory '/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc/4a52' Handle the file: 0x200004a51:0x3:0x0 Remove non-cached file: /mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc/4a52/0x200004a51:0x3:0x0 flags: 1 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d36a.sanity-pcc is idle now PASS 36a (5s) == sanity-pcc test 36b: Stale RO-PCC copy should be deleted after remove the PCC backend ========================================================== 23:50:56 (1746071456) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0872461 s, 601 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d36b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 7dc41f16-c7da-4ac7-a7bd-596862a04dfe Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f36b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=7dc41f16-c7da-4ac7-a7bd-596862a04dfe (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' check file 0:'4a54' under directory '/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc' check file 1:'0x200004a51:0x5:0x0' under directory '/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc/4a54' Handle the file: 0x200004a51:0x5:0x0 Remove non-cached file: /mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc/4a54/0x200004a51:0x5:0x0 flags: 1 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d36b.sanity-pcc is idle now PASS 36b (3s) == sanity-pcc test 37: Multiple readers on a shared file with PCC-RO mode ========================================================== 23:50:59 (1746071459) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588663 s, 891 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d37.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: d47e27fe-985c-4ad0-b9f6-078df2206118 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f37.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d47e27fe-985c-4ad0-b9f6-078df2206118 (extents) (64bit) (large files) (huge files) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819885 s, 639 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f37.sanity-pcc on /mnt/pcc.d37.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) umount: /mnt/pcc.d37.sanity-pcc.2: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f37.sanity-pcc on /mnt/pcc.d37.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 60c072a4-ed9e-4b8c-8608-d421f5a08566 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f37.sanity-pcc.2: Linux rev 1.0 ext4 filesystem data, UUID=60c072a4-ed9e-4b8c-8608-d421f5a08566 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d37.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: a0 autocache: projid={2} pcc: - pccpath: /mnt/pcc.d37.sanity-pcc.2 hsmtool: posix_v2 rwid: 0 roid: 2 flags: a0 autocache: projid={2} file: /mnt/lustre/d37.sanity-pcc/f37.sanity-pcc, type: readonly, PCC_file: /4a56/0x200004a51:0x7:0x0, open_count: 0, flags: 0 file: /mnt/lustre2/d37.sanity-pcc/f37.sanity-pcc, type: readonly, PCC_file: /4a56/0x200004a51:0x7:0x0, open_count: 0, flags: 0 /mnt/pcc.d37.sanity-pcc.2 is idle now /mnt/pcc.d37.sanity-pcc is idle now PASS 37 (7s) == sanity-pcc test 38: Verify LFS pcc state does not trigger prefetch for auto PCC-RO ========================================================== 23:51:06 (1746071466) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg311-client.virtnet /mnt/lustre (opts:) Stopping client oleg311-client.virtnet /mnt/lustre opts: Stopping clients: oleg311-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg311-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg311-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg311-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg311-server' oleg311-server: oleg311-server.virtnet: executing load_modules_local oleg311-server: Loading modules from /home/green/git/lustre-release/lustre oleg311-server: detected 4 online CPUs by sysfs oleg311-server: MODOPTS_LIBCFS= oleg311-server: Force libcfs to create 2 CPU partitions oleg311-server: libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory oleg311-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a612e000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012c857800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a612e000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012c857800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0769227 s, 682 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d38.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: ee45425a-5a47-4740-a38c-7551a599b685 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f38.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=ee45425a-5a47-4740-a38c-7551a599b685 (extents) (64bit) (large files) (huge files) file: /mnt/lustre/d38.sanity-pcc/f38.sanity-pcc, type: none QQQQQ Cleanup PCC backend on /mnt/lustre /mnt/pcc.d38.sanity-pcc is idle now PASS 38 (63s) == sanity-pcc test 39: Test Project quota on loop PCC device ========================================================== 23:52:09 (1746071529) SKIP: sanity-pcc test_39 Not support project quota on local filesystem SKIP 39 (2s) == sanity-pcc test 40: Test async open attach in the background for PCC-RO file ========================================================== 23:52:11 (1746071531) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg311-client.virtnet /mnt/lustre (opts:) Stopping client oleg311-client.virtnet /mnt/lustre opts: Stopping clients: oleg311-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg311-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg311-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg311-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg311-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg311-server' oleg311-server: oleg311-server.virtnet: executing load_modules_local oleg311-server: Loading modules from /home/green/git/lustre-release/lustre oleg311-server: detected 4 online CPUs by sysfs oleg311-server: MODOPTS_LIBCFS= oleg311-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg311-server: oleg311-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg311-client: oleg311-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Starting client oleg311-client.virtnet: -o user_xattr,flock 192.168.203.111@tcp:/lustre /mnt/lustre2 Started clients oleg311-client.virtnet: 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800aadd2000.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800aae60000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800aadd2000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800aae60000.idle_timeout=debug disable quota as required 200+0 records in 200+0 records out 209715200 bytes (210 MB) copied, 0.382309 s, 549 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d40.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/204800 done Creating filesystem with 204800 1k blocks and 51200 inodes Filesystem UUID: 4e8872d3-8b27-4b24-938f-1439eb61b575 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/25 done Writing inode tables: 0/25 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/25 done /tmp/f40.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=4e8872d3-8b27-4b24-938f-1439eb61b575 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d40.sanity-pcc/d40.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={100} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.489926 s, 107 MB/s 100 - /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none ldlm.namespaces.lustre-OST0000-osc-ffff8800aadd2000.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff8800aae60000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800aadd2000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800aae60000.lru_size=clear osc.lustre-OST0000-osc-ffff8800aadd2000.stats=clear osc.lustre-OST0000-osc-ffff8800aae60000.stats=clear osc.lustre-OST0001-osc-ffff8800aadd2000.stats=clear osc.lustre-OST0001-osc-ffff8800aae60000.stats=clear fail_val=1 fail_loc=0x214 llite.lustre-ffff8800aadd2000.pcc_async_threshold=5MB llite.lustre-ffff8800aae60000.pcc_async_threshold=5MB Test open attach with pcc_async_threshold=5MB 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 2.03277 s, 516 kB/s Before: 0 After: 2 file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none, flags: attaching /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc | grep -E -c 'type: readonly' Waiting 50s for '1' Updated after 8s: want '1' got '1' file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff8800aadd2000.pcc_async_threshold=1G llite.lustre-ffff8800aae60000.pcc_async_threshold=1G ldlm.namespaces.lustre-OST0000-osc-ffff8800aadd2000.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff8800aae60000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800aadd2000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800aae60000.lru_size=clear osc.lustre-OST0000-osc-ffff8800aadd2000.stats=clear osc.lustre-OST0000-osc-ffff8800aae60000.stats=clear osc.lustre-OST0001-osc-ffff8800aadd2000.stats=clear osc.lustre-OST0001-osc-ffff8800aae60000.stats=clear Test open attach with async_threshold=1G file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: readonly, PCC_file: /d40.sanity-pcc/59f3/0x2000059f1:0x2:0x0, open_count: 0, flags: 0 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.00746256 s, 141 MB/s Before: 50 After: 50 Time1: 2 Time2: 10 file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff8800aadd2000.pcc_async_threshold=5MB llite.lustre-ffff8800aae60000.pcc_async_threshold=5MB ldlm.namespaces.lustre-OST0000-osc-ffff8800aadd2000.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff8800aae60000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800aadd2000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800aae60000.lru_size=clear Read 1MB data with async_threshold=5MB 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 1.14015 s, 920 kB/s /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc | grep -E -c 'type: readonly' Waiting 50s for '1' Updated after 10s: want '1' got '1' file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff8800aadd2000.pcc_async_threshold=1G llite.lustre-ffff8800aae60000.pcc_async_threshold=1G ldlm.namespaces.lustre-OST0000-osc-ffff8800aadd2000.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff8800aae60000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800aadd2000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800aae60000.lru_size=clear Read 1MB data with async_threshold=1G 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.00400465 s, 262 MB/s Time1: 1 Time2: 10 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d40.sanity-pcc is idle now PASS 40 (92s) == sanity-pcc test 41: Test mtime rule for PCC-RO open attach with O_RDONLY mode ========================================================== 23:53:43 (1746071623) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756633 s, 693 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d41.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 9bb23122-017d-488a-9799-033bcccb1bec Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f41.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=9bb23122-017d-488a-9799-033bcccb1bec (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d41.sanity-pcc/d41.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: mtime>{1m} pcc_ro_data pcc_ro_data Cleanup PCC backend on /mnt/lustre /mnt/pcc.d41.sanity-pcc is idle now PASS 41 (5s) == sanity-pcc test 42: PCC attach without attach ID specified ========================================================== 23:53:48 (1746071628) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.106107 s, 593 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d42.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: ed3835c7-7706-4774-b16a-41d90ffa49ef Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f42.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=ed3835c7-7706-4774-b16a-41d90ffa49ef (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d42.sanity-pcc/d42.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={100} file: /mnt/lustre/f42.sanity-pcc, type: readonly, PCC_file: /d42.sanity-pcc/59f5/0x2000059f1:0x4:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d42.sanity-pcc is idle now PASS 42 (4s) == sanity-pcc test 43: Auto attach at open() should add capacity owner check ========================================================== 23:53:52 (1746071632) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.1166 s, 540 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d43.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 65c42a3b-10d8-4540-a92b-23a978078af3 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f43.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=65c42a3b-10d8-4540-a92b-23a978078af3 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d43.sanity-pcc/d43.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: size<{100M} file: /mnt/lustre/f43.sanity-pcc, type: none running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f43.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f43.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/f43.sanity-pcc] file: /mnt/lustre/f43.sanity-pcc, type: none Cleanup PCC backend on /mnt/lustre /mnt/pcc.d43.sanity-pcc is idle now PASS 43 (5s) == sanity-pcc test 44: Verify valid auto attach without re-fetching the whole files ========================================================== 23:53:57 (1746071637) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.11503 s, 547 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d44.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 02e0cbc7-e41f-491d-b70e-cf6dacb98be8 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f44.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=02e0cbc7-e41f-491d-b70e-cf6dacb98be8 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d44.sanity-pcc/d44.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800aadd2000.pcc_async_threshold=0 llite.lustre-ffff8800aae60000.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.531669 s, 98.6 MB/s /mnt/lustre/f44.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x6 lmm_fid: [0x2000059f1:0x6:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 420 0x1a4 0x240000400 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300238 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065267 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0828502 s, 633 MB/s 52428800 bytes (52 MB) copied, 0.102698 s, 511 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.349507 s, 150 MB/s , 0.335189 s, 156 MB/s 52428800 bytes (52 MB) copied, 0.340581 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.382241 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.350416 s, 150 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.397574 s, 132 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.415827 s, 126 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.433394 s, 121 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152095 s, 345 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.266264 s, 197 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.457955 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.281721 s, 186 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.283089 s, 185 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.48262 s, 109 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.522005 s, 100 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.530206 s, 98.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297476 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235343 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225195 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020421 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246598 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038623 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484986 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.076948 s, 681 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431034 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623606 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352332 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740344 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0941466 s, 557 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615292 s, 852 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0734017 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041035 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474456 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220938 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288881 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346444 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220411 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329862 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303189 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333498 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333316 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078596 s, 667 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169741 s, 3.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0332713 s, 1.6 GB/s , 0.0259802 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473924 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293428 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532187 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439287 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438098 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.079666 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.084528 s, 620 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101098 s, 519 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321678 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.117422 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464159 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072679 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.082985 s, 632 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274808 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277787 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195671 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520511 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351208 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479625 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229643 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065675 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626607 s, 837 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819311 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532865 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587412 s, 893 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123982 s, 423 MB/s 52428800 bytes (52 MB) copied, 0.0505112 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110316 s, 475 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0942879 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402006 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379741 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444213 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246121 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152443 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583354 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052347 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222436 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544341 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162764 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384711 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417993 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479445 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261587 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035558 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0871622 s, 602 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0875491 s, 599 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471385 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257978 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284118 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351142 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036179 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253324 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378955 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364284 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369939 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506053 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207736 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042454 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333729 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325479 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337942 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530549 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646276 s, 811 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0922843 s, 568 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0962053 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664431 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507286 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0852885 s, 615 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258397 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651722 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228495 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373137 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403985 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253718 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233522 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215712 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520175 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286861 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260169 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269981 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0788759 s, 665 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338241 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218643 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349284 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470755 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044278 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358513 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476743 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184367 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377108 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210496 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370889 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524634 s, 999 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170414 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433327 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277345 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272389 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640072 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240445 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479716 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039671 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579675 s, 904 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493328 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0788568 s, 665 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318492 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533645 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347312 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325277 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205747 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261775 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0210552 s, 2.5 GB/s , 0.0385028 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318755 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246718 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365772 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206175 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221867 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038175 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313108 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330224 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305812 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024051 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284917 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469464 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355578 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280484 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036702 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365271 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332459 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242135 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.068105 s, 770 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340275 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384106 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470124 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0148014 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611194 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262807 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207026 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181507 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390913 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03203 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039903 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027261 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255267 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290784 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342584 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344406 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501802 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214384 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294735 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027558 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0684938 s, 765 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431209 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489527 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057405 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0670593 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304039 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341181 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259514 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279791 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340701 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247055 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171982 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395504 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300121 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303742 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298518 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334155 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482235 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038463 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407347 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0704791 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248465 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465941 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211715 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333379 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541139 s, 969 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609006 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377019 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047598 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139308 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740545 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1064 s, 493 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487633 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310964 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0808789 s, 648 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016945 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0712446 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100234 s, 523 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0924785 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488312 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273758 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276314 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270302 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281994 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352043 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305588 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343566 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526946 s, 995 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355037 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416192 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200038 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224236 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445614 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.068427 s, 766 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327181 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0629019 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618038 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0997666 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532279 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0996362 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0883751 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288549 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310976 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286538 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487115 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292137 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041153 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188422 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254477 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630216 s, 832 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364534 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183177 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333223 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163759 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278874 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283897 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230888 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635952 s, 824 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033252 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02591 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329908 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619478 s, 846 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274336 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258702 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577564 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048359 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304674 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486807 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170986 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428727 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256377 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298303 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184056 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358668 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350185 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324125 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311698 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261691 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466408 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547312 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453797 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279752 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274516 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349856 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031667 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351258 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413348 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397709 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229655 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0205336 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423521 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180024 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337967 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335674 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322295 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384478 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316025 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160889 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485407 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291071 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732982 s, 715 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675439 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296333 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0713973 s, 734 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581406 s, 902 MB/s 52428800 bytes (52 MB) copied, 0.0551124 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265605 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455475 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352131 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252341 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482932 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03224 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593764 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427968 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601984 s, 871 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035221 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252641 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301552 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05056 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233975 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027663 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281277 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386518 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327319 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257548 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679985 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402695 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238014 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513218 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461434 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250313 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043681 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291157 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311018 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273961 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195183 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016134 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218741 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235164 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226503 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292877 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307586 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224972 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158206 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499559 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0172509 s, 3.0 GB/s , 0.0214532 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299275 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252167 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289987 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282886 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026784 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230331 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280079 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283686 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312736 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208456 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237045 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379323 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191945 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624511 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363016 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032224 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402888 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225825 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271622 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327217 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187593 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253808 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224156 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058691 s, 893 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292114 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643697 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077507 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381708 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.092892 s, 564 MB/s , 0.0550557 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718387 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111677 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298123 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405761 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445854 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0975931 s, 537 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385661 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382783 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392764 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194706 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342031 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024298 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299406 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299591 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305352 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042636 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395378 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181084 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023985 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018977 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252689 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447951 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358258 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328601 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040399 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508267 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0286743 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342166 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152735 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609985 s, 860 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457596 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020944 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.045948 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388978 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256321 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279454 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348961 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289731 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343308 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364729 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238912 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425206 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301016 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045091 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528837 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299575 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340547 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248845 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200476 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464515 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205799 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289012 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679088 s, 772 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074875 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10248 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0863509 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.10101 s, 519 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124158 s, 422 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12361 s, 424 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667627 s, 785 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1274 s, 412 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0654958 s, 800 MB/s 52428800 bytes (52 MB) copied, 0.0783209 s, 669 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0674003 s, 778 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448244 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271365 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396284 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413898 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540828 s, 969 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0894927 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455873 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313543 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0645845 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579255 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440712 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701745 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489289 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032669 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02013 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378475 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326194 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0372847 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416858 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365531 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385521 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255586 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280909 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233782 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491934 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268979 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381167 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235989 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198615 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243665 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228362 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321722 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229035 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029134 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426477 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176319 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371251 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329604 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331707 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208042 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272426 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179696 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513198 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040788 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039975 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222467 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269753 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349598 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039788 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155733 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265858 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170359 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248589 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264765 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285857 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328354 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0260073 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303216 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273841 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296181 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0223638 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0241595 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366045 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330195 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033617 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270127 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453092 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234085 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253376 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0197383 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337555 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400814 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019817 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256943 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386788 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494888 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155422 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018514 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630728 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293266 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381786 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147034 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162621 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043933 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038245 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244564 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333416 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334869 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403484 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349982 s, 1.5 GB/s , 0.0385411 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268426 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325706 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01988 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332329 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444118 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402063 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230195 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277817 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322943 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307073 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301506 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036448 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408826 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537078 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216288 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249617 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211181 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025262 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032052 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363833 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422054 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250279 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436185 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250502 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237837 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534144 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416372 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450021 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240696 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283564 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221611 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0247878 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212775 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234638 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195987 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254444 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222821 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328457 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251665 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253645 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260119 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271808 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403927 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213091 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230815 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347791 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169843 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490606 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561155 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455143 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0720421 s, 728 MB/s 52428800 bytes (52 MB) copied, 0.0710873 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245849 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120885 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080603 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510447 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0880338 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286544 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255888 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694454 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0990562 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165266 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577203 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0784508 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155116 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203803 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437987 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023071 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211488 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388037 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249243 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025262 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259346 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490176 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025295 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268101 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375485 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408972 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287567 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418302 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495242 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421726 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352666 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635564 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381863 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423718 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230737 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326654 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334478 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201208 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525254 s, 998 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449891 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019167 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331636 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341125 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023382 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240224 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316698 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374717 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295655 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477478 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310574 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150109 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191642 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326911 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366314 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0698269 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543448 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380755 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152289 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209163 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203431 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359226 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271335 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382617 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250321 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337026 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190025 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175602 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027432 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355109 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176274 s, 3.0 GB/s , 0.049424 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329505 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326059 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415731 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0176262 s, 3.0 GB/s , 0.0190164 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033343 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248681 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218604 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255823 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255302 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261244 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027792 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170012 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181147 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280473 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273206 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427198 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280006 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023738 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175553 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452773 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444991 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321413 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286396 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0712319 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429691 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194335 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491399 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668797 s, 784 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208499 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480636 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399835 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419465 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401729 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216672 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247304 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037238 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389339 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193027 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230202 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321736 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322999 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0254492 s, 2.1 GB/s , 0.0254312 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320895 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204189 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210768 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0180284 s, 2.9 GB/s , 0.0193606 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195578 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293045 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217614 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256272 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367693 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0341444 s, 1.5 GB/s , 0.0252621 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316929 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021855 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238956 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03141 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243827 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298641 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238687 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025756 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303643 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219743 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345121 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355238 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240656 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342212 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261353 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239106 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473602 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389217 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609421 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061409 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299872 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328871 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551408 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035492 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394225 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452699 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471079 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340842 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247363 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350171 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0772669 s, 679 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460212 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555503 s, 944 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418509 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581927 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666077 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497888 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524464 s, 1000 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625001 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435324 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672331 s, 780 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031391 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251847 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369386 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461491 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258843 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288729 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225199 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259957 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200291 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199539 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206571 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0249602 s, 2.1 GB/s , 0.0241195 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453238 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380648 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497591 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475678 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241087 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184874 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259537 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225475 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331339 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459879 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695483 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280768 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.087253 s, 601 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316946 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182449 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349539 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370995 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311693 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307454 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377853 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263076 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276519 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474873 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426161 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0815109 s, 643 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294052 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329109 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037951 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409329 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323886 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030317 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230884 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282222 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025509 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259914 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328171 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355074 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366162 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193701 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043057 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241354 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242845 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018418 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197883 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282646 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341823 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260399 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523103 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045023 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047871 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329865 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356372 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408194 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327329 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549964 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534418 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240105 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167771 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213602 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180344 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234986 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647807 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231081 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357881 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263883 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511368 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034233 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326453 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521846 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205232 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323157 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327539 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236568 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298322 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060261 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367698 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387099 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356089 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450839 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224873 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041538 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363517 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508489 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596952 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044023 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398586 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417136 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644367 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0698616 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558198 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047742 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528199 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575367 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323239 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351862 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385636 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347782 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373097 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294913 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330022 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245322 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0244294 s, 2.1 GB/s , 0.0216936 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023795 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302708 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171939 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204159 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396212 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562213 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047391 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0803222 s, 653 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840699 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781768 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505811 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041867 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634306 s, 827 MB/s 52428800 bytes (52 MB) copied, 0.0580546 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710125 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510061 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353417 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0958304 s, 547 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036264 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363567 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165609 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417025 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498433 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316031 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248363 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416867 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440347 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393418 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408268 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346466 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194192 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569677 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332659 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345315 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031147 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687542 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237286 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613488 s, 855 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294637 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528833 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195313 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301474 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457832 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461604 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293229 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599664 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502604 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426708 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579621 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673166 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584646 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775753 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715352 s, 733 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027717 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382358 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360209 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505309 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485211 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746351 s, 702 MB/s 52428800 bytes (52 MB) copied, 0.0367618 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518048 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416408 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201581 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286437 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286172 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286472 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271587 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380297 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290383 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321994 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279731 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289008 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324287 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308043 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192738 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212006 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265061 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306576 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269055 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211405 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209744 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271498 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424062 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206609 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310422 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211829 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230511 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399708 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308229 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413596 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378718 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276382 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285417 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258948 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218343 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288851 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264771 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295745 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262931 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356144 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035055 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303704 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383512 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191984 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302987 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412213 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395703 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454254 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237877 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216524 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184504 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370069 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535368 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377377 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275095 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352289 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020428 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300361 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246578 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176724 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199059 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359529 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374275 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423565 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240102 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215056 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179509 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375926 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237616 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199895 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024645 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240691 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276621 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298249 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400472 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550503 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284716 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327258 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227832 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200802 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293198 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028607 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431451 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277832 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446653 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334905 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0431541 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392105 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034971 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061846 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526364 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439094 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638229 s, 821 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612747 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064591 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488763 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184161 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439098 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422234 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633893 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029604 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267608 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019714 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201527 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504685 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02137 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495119 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263176 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421936 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249201 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02513 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309916 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0925624 s, 566 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689913 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259882 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250867 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018215 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321246 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317208 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367951 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259861 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307485 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281293 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510396 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292841 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330814 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419015 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289563 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343025 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164443 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292781 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565299 s, 927 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315478 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02458 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020089 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024398 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351641 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419547 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442081 s, 1.2 GB/s , 0.0530898 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189645 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202235 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413091 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242769 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257082 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040987 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493504 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360525 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662961 s, 791 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0928258 s, 565 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115004 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272244 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429014 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116703 s, 449 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050206 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341722 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216082 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340221 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312193 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043559 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490386 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024478 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630087 s, 832 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256247 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254124 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374393 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273116 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258255 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338397 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252958 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237632 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298428 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0520854 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281002 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0669234 s, 783 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284009 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328531 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203755 s, 2.6 GB/s , 0.0347155 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260484 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285166 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030656 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209783 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644461 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05195 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035371 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400196 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318442 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260508 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222706 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285701 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331091 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295711 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261248 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270684 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367094 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389929 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295472 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294078 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195454 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266373 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226029 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0302853 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242257 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235351 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356323 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319206 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293574 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261849 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486314 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270203 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300533 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360827 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269082 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179811 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490469 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318178 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349795 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314231 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366786 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299822 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344124 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451052 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293696 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260139 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194182 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605908 s, 865 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540109 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639745 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0858545 s, 611 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0903902 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549556 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0930787 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342109 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189775 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027351 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381848 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513535 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255597 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868911 s, 603 MB/s 52428800 bytes (52 MB) copied, 0.0505115 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350996 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0821922 s, 638 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716146 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483917 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477503 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587848 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671585 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519255 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247206 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486497 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376475 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316659 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191954 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202356 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383705 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409141 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238461 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026672 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298182 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260998 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384041 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318112 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413409 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266465 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02467 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364294 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353716 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221131 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292833 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431134 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278451 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163662 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344221 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248535 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250032 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040622 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026714 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191011 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489965 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166513 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409269 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490669 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606884 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069857 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169417 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03394 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204562 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218693 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225873 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182056 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050596 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320449 s, 1.6 GB/s , 0.0195385 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425157 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188235 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440462 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022646 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229263 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461945 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302668 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481964 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204315 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349648 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276073 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.02916 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315794 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234109 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347732 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577135 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226334 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410112 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194604 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201122 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394221 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343989 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408302 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225723 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249773 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417804 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319251 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254954 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336358 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235504 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203382 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389831 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402753 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359788 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508486 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330012 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219963 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367063 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536987 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488642 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024732 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441737 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447728 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513343 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579128 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233937 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0650428 s, 806 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278303 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262414 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160789 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283946 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279715 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255305 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028451 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322523 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184773 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372421 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316511 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184797 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400224 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233061 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316473 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172801 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204459 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243096 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180037 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181485 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253535 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029104 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298782 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249857 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199654 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192183 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198101 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251333 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271649 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269847 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202675 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196072 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399811 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460333 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051033 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587534 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0505465 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0177155 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473727 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234209 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359434 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200179 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251611 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241556 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242496 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017575 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221084 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364147 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533385 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187512 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266633 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296599 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451492 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588963 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265167 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407813 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0517767 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0454361 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576916 s, 909 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623665 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505406 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035073 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0921905 s, 569 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0647491 s, 810 MB/s 52428800 bytes (52 MB) copied, 0.0508126 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410119 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185215 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355685 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269032 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292727 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302503 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215601 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444094 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234941 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277538 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215999 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215355 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291056 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312767 s, 1.7 GB/s , 0.0250689 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047261 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329007 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284702 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018101 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203384 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189065 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160794 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424716 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192748 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309944 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320876 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357693 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475533 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024258 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255014 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451164 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152317 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222316 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207449 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389542 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533528 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546911 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624318 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619883 s, 846 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374077 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388033 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197672 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200018 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241465 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312371 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451612 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227282 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272686 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251811 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259819 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423057 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433583 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215251 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188447 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155601 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020127 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240475 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193149 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335575 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244933 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192595 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240386 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273118 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469056 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465987 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507782 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231034 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197925 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297186 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351791 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348118 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297188 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213203 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251695 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202224 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363078 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304891 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191939 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236606 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209505 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200809 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191621 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290755 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197208 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241183 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031557 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346462 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328469 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315056 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334536 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218065 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036251 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277091 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175376 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188279 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416009 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181031 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301375 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401982 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019589 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370138 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270124 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310538 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337235 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179605 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216298 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332528 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420132 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398075 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209778 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222663 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236328 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266893 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222529 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028751 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285344 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339689 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270408 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337295 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292743 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190142 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0200651 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316471 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314157 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299848 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219087 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173271 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364672 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035224 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0209861 s, 2.5 GB/s , 0.0431868 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359835 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373099 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175519 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337911 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225647 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234258 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363391 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257595 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367738 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210842 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208457 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232078 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275398 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214953 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287302 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222064 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024026 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180177 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231637 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201408 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293585 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248866 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361041 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422879 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482643 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498486 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224588 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367375 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361969 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048685 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378326 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686293 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700869 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372689 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503202 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591966 s, 886 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379964 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0638242 s, 821 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718256 s, 730 MB/s 52428800 bytes (52 MB) copied, 0.0257583 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440973 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0714715 s, 734 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463363 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440525 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345485 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.06791 s, 772 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533556 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257093 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411673 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260133 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0512917 s, 1.0 GB/s , 0.0627485 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.06203 s, 845 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0669044 s, 784 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078748 s, 666 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204203 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237871 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375439 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625575 s, 838 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518051 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408552 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390315 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566243 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555795 s, 943 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600021 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019607 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389933 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208672 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057174 s, 917 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0900609 s, 582 MB/s 52428800 bytes (52 MB) copied, 0.049596 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648731 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442355 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437994 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276857 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310281 s, 1.7 GB/s , 0.0391329 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0312423 s, 1.7 GB/s , 0.0265407 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314528 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381489 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048372 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291377 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335144 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287644 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521861 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214318 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264975 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03916 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429522 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551216 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340296 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552557 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497688 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355552 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366884 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339374 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495212 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338098 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372859 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052248 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810259 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270094 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763322 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464387 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451367 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378874 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040966 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345248 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417567 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581649 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065268 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0855986 s, 612 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273842 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297658 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287903 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542315 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350872 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339263 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198948 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557865 s, 940 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056192 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0972755 s, 539 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647 s, 810 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135384 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753584 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575215 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468676 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157805 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188631 s, 2.8 GB/s , 0.16294 s, 322 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125976 s, 416 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323947 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034544 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289387 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578495 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266761 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381917 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427412 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479799 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223777 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344662 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311032 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187325 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235883 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433693 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348029 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511136 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595948 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284161 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180816 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313808 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353418 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578421 s, 906 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376802 s, 1.4 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256108 s, 2.0 GB/s , 0.0653526 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426699 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458181 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239226 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269916 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575894 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738395 s, 710 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420729 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465151 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548424 s, 956 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522299 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509148 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427017 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401008 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582922 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584279 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278326 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269738 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416373 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268265 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254426 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400543 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326293 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725426 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471392 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438597 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265829 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275718 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324387 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209635 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227414 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0825256 s, 635 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337325 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293049 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345433 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294753 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385183 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042705 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430829 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344118 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034751 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397738 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282015 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540178 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777282 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0859604 s, 610 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103168 s, 508 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295267 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529232 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572424 s, 916 MB/s , 0.0534146 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368041 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517669 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209322 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397808 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585822 s, 895 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456317 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868103 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537931 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351126 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463862 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314331 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738868 s, 710 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400092 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423656 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325256 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325353 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0545835 s, 961 MB/s , 0.0802961 s, 653 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106295 s, 493 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.091044 s, 576 MB/s 50+0 records in 50+0 records out , 0.0955598 s, 549 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0846555 s, 619 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528868 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598879 s, 875 MB/s 52428800 bytes (52 MB) copied, 0.0871587 s, 602 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401721 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489614 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.126632 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103701 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100172 s, 523 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296849 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219036 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265756 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403884 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341648 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328474 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029348 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280662 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044536 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236834 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254259 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482935 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382024 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0214968 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042349 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516129 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0719932 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386207 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442132 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453928 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044201 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286908 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435157 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373199 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0674192 s, 778 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427521 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221719 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229531 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033565 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240668 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219273 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227947 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259932 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449284 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557251 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190436 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602993 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610784 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363263 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.06154 s, 852 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046316 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259654 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566561 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207651 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371301 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465289 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560584 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231213 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429709 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444393 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498023 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210577 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467846 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383698 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360245 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463647 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535449 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688824 s, 761 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.06678 s, 785 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220225 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332479 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316329 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0645097 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504641 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040038 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415641 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232394 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183357 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156998 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171263 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256185 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223341 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284767 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267091 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335224 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191615 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038722 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303732 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297029 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182965 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609239 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346953 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296185 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05877 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025687 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219886 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431376 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602895 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307551 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042995 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187941 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221961 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373246 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483734 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195467 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265334 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033198 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035071 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339652 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242933 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616806 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326929 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0674596 s, 777 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625894 s, 838 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250064 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0861334 s, 609 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038442 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0768085 s, 683 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380303 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15343 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428184 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114394 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122861 s, 427 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666709 s, 786 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312005 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339243 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439301 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355081 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052998 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453689 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491715 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526993 s, 995 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381058 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0874077 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329366 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520171 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289842 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372611 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038463 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361091 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311312 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282915 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0413966 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044053 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491825 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546097 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577361 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293955 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183831 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290372 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323452 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185269 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043342 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271583 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248955 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209645 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231539 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026689 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181091 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351521 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262266 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249308 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299628 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200687 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311362 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425938 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553236 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251806 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346758 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031558 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341719 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263507 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198526 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242261 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206616 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276425 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225293 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352245 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377196 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233504 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188526 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295879 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188072 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219667 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288752 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219258 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254249 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339272 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449919 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205643 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287212 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190595 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0198722 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0173827 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210481 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018775 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188028 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237274 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173734 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293709 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371818 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217172 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319139 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367794 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250374 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02886 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024793 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351669 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338767 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307981 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038832 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241064 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222392 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024801 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411069 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413033 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249333 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233824 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266153 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206386 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313457 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0354663 s, 1.5 GB/s , 0.0287424 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470672 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209514 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328333 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208653 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390475 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171479 s, 3.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.01968 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689688 s, 760 MB/s , 0.0352869 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300603 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347578 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285102 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386629 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281502 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467834 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331324 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405842 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467462 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0684033 s, 766 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552261 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387994 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500504 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296564 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321151 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368779 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170821 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035442 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250195 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261082 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441522 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354933 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205313 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167142 s, 3.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.030326 s, 1.7 GB/s , 0.0231385 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200141 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244308 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500771 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320667 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024349 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345281 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240653 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282536 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045637 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308124 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486658 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046242 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217088 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288658 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244157 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323444 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271544 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286599 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422428 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157312 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278984 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382399 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253185 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310749 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416575 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031365 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392071 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475843 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260858 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210937 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322465 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468243 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368148 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237856 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323645 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242564 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263064 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189346 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171586 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168613 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303919 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328865 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043302 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194974 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174319 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040045 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508901 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345556 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509163 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200409 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426277 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344776 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315316 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0300368 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344134 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019673 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269813 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021335 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383764 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249202 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358806 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469326 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177 s, 3.0 GB/s , 0.0339413 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327313 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229478 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262008 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261245 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421255 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040879 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219474 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216096 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255957 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206459 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199084 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183641 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234112 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219921 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189692 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302465 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349535 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188433 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240234 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255342 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329845 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318282 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226662 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298492 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235146 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215446 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323062 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376589 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301329 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394716 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232879 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425587 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220401 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249647 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212561 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190815 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192274 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018567 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271984 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291559 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166519 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213889 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205122 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195208 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187429 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159822 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171925 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207164 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185852 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201045 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153078 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288821 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182838 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203261 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288638 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235122 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173579 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159247 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173318 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335687 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234926 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156148 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220008 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193724 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356305 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165623 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195349 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191373 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267195 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242192 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266784 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267796 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179005 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152935 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183371 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205225 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233572 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294774 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260201 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223621 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337872 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362895 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405957 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029541 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0180705 s, 2.9 GB/s , 0.0169273 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176195 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291395 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178688 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0387558 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190425 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265137 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0439661 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287844 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476804 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231166 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230178 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368894 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034617 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389509 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446771 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207634 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193224 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206962 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216065 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241084 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214014 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313343 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034985 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031387 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387932 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200106 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371002 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0192055 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0195873 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280995 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242641 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251688 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230022 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219341 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202056 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216687 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189249 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174612 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230814 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302841 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394671 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304583 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377056 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416586 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230358 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162031 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162905 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219738 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277142 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444159 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300643 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411386 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359128 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318167 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309009 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310663 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450377 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323141 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293995 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160357 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176893 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183144 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205519 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154436 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300886 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296334 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341445 s, 1.5 GB/s , 0.0299014 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311549 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198487 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214181 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316781 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515887 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0638041 s, 822 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641118 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462094 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069678 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352677 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731861 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377369 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298456 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513711 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313709 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312858 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286116 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266693 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273272 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209534 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213046 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261226 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224088 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039514 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242112 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594436 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361558 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026431 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285844 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436544 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036581 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410201 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422151 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028315 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201976 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433251 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407636 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198603 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169862 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278254 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315129 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267515 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368587 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325505 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430471 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395001 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311212 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419534 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553455 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571795 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225741 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381692 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400164 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060267 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026699 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223488 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243377 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463183 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380885 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280312 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024551 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042144 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281243 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026776 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041679 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467116 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543881 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588676 s, 891 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379199 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272129 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232728 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179011 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05457 s, 961 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0299175 s, 1.8 GB/s , 0.0245194 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289004 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343556 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274088 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435674 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403005 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0712392 s, 736 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0400596 s, 1.3 GB/s , 0.0242291 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430277 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365445 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338799 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323604 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183817 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359942 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192883 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283857 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250641 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0336375 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492088 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491781 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350536 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208025 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435608 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387278 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036881 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456766 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246098 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0335048 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259272 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273528 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348038 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440353 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597054 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328849 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019586 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178308 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210251 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199969 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304006 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241102 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380283 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018089 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295793 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496798 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331811 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368293 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486278 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036486 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361653 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253425 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435912 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232647 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266991 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328464 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396957 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021755 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247621 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258001 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288019 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018834 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039391 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510797 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511569 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172298 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391478 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476195 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101755 s, 515 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397695 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318899 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334731 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495185 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447717 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641952 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026145 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028078 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441342 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228612 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361875 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483997 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605624 s, 866 MB/s , 0.0236344 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661079 s, 793 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157732 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0645288 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615322 s, 852 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199988 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488657 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106384 s, 493 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320085 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585614 s, 895 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577211 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463895 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402016 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340977 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282924 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394333 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290496 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374436 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562365 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362136 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453362 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187865 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283995 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365503 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442137 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381318 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176262 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466205 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368505 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599107 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494364 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434354 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0892566 s, 587 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0890104 s, 589 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352853 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192112 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426726 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0751117 s, 698 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383604 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0755839 s, 694 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462187 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256724 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528017 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331271 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382379 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399095 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297109 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263328 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316183 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237318 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217762 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352759 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312864 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251545 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618398 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257545 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490844 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541839 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604446 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334696 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204259 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292688 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220431 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027183 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584007 s, 898 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247285 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244788 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475118 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620406 s, 845 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063266 s, 829 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321979 s, 1.6 GB/s , 0.0894818 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304872 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708985 s, 739 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631578 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445051 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563071 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180749 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353035 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240481 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355877 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287611 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211819 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281238 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366116 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314541 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255249 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382881 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403343 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033166 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392944 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532613 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224595 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253597 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283628 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385017 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308853 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232517 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543626 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177638 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476283 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283018 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031564 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238534 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340707 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257259 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350662 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360576 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173988 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456517 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367483 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454171 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727331 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187172 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444329 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479723 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202135 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312877 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409328 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217665 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351872 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389178 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244121 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356897 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225052 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271016 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364837 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280048 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177233 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371072 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257277 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487447 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034078 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0624212 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572495 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614296 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675482 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219958 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395272 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243714 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030548 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429502 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.083425 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550978 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415064 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531786 s, 986 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578682 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0755622 s, 694 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586696 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588902 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630655 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279938 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0684492 s, 766 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378778 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080288 s, 653 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415421 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322371 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445345 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0751551 s, 698 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716477 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189555 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041293 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375545 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789981 s, 664 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265643 s, 2.0 GB/s , 0.040285 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522158 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220991 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416069 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278876 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238569 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294312 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0235785 s, 2.2 GB/s , 0.0243196 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279552 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423081 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295678 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397892 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256227 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279684 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233371 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375431 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258959 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276524 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359961 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271889 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051872 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329901 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541543 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0923529 s, 568 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393873 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461095 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521634 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687192 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159005 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167052 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111098 s, 472 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390013 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485001 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0830891 s, 631 MB/s , 0.0175285 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330416 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488961 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047735 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328969 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489893 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306905 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227827 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241371 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226698 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234994 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446116 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218088 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243977 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422939 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247307 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294848 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274959 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283631 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306039 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307753 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236245 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182662 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286308 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020568 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225534 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245698 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394223 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226712 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350335 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341799 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262421 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250436 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312182 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0175681 s, 3.0 GB/s , 0.0175622 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492738 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499004 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265843 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410765 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020676 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289732 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304095 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371251 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488938 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514819 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492001 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531094 s, 987 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222927 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330872 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234123 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314542 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196583 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276653 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334277 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201192 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291665 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434063 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239416 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310973 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337403 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345536 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575087 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250342 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275918 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037749 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341448 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195679 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230863 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246915 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195117 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302141 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222796 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595499 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501936 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043843 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276145 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255244 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198261 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313962 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196223 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018902 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252887 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189339 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266084 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02982 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347433 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452151 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023139 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257005 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306915 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320898 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450029 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715361 s, 733 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494454 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243157 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255839 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309339 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249938 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0569603 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266317 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253221 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404691 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296936 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247726 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309261 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221389 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211342 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023925 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517705 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532277 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040264 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332415 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478853 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509308 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486464 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361247 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256932 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699664 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0455768 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0477663 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070134 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070162 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654685 s, 801 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622029 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037718 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392161 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047053 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.059689 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429666 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461135 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020843 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307262 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586694 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0650556 s, 806 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173717 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418035 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377787 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265793 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208188 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360018 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399291 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353074 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402637 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384108 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229485 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434002 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420449 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314395 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309019 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294085 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057351 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664993 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0920537 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.07853 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316542 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355577 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424776 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303301 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224875 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412064 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472928 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219329 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315243 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628384 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258949 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715676 s, 733 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478187 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353687 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502664 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600795 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03279 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377578 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328941 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518937 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540077 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0750546 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447815 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370328 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586761 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639434 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563907 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526364 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0883681 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033575 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027741 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248303 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473208 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657052 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742965 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05828 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274739 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232526 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02262 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462975 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320996 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381259 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437602 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497014 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559143 s, 938 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631945 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456264 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0692236 s, 757 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264457 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527303 s, 994 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352856 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220419 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826341 s, 634 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336908 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0439814 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325348 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255733 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556081 s, 943 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391542 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415366 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346992 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442052 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342381 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411413 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496465 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325393 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373409 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355778 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040525 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569305 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355758 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487695 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041518 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603138 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580195 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0844887 s, 621 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271295 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0690557 s, 759 MB/s 52428800 bytes (52 MB) copied, 0.0770528 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659781 s, 795 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683664 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0960084 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033048 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395955 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313785 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361917 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387925 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337179 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299489 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424436 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229313 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452246 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035522 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263924 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375975 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410929 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329175 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443453 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518196 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255582 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312131 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236075 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361757 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286984 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443701 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350176 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265602 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583289 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201009 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560542 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369031 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0934549 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113377 s, 462 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139246 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263983 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254682 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530953 s, 987 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317811 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324987 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531003 s, 987 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341748 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509145 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428546 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532138 s, 985 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0275356 s, 1.9 GB/s , 0.0278489 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265164 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335112 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364828 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344469 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194018 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317769 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267856 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029013 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305847 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203102 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525709 s, 997 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314448 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687326 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039618 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488747 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417909 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237704 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335036 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388076 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326091 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337668 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673363 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577638 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599652 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427634 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375614 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413107 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404098 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236864 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370183 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519504 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374593 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316026 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423704 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365968 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296671 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316566 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326319 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385261 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04128 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281289 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225983 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344217 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354046 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351933 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279777 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489344 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578636 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041351 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456777 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213485 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294478 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039391 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423395 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337493 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450933 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229064 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537173 s, 976 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.085411 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0835027 s, 628 MB/s 52428800 bytes (52 MB) copied, 0.0740306 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0790422 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340778 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597411 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503089 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0873812 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457893 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031149 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0803745 s, 652 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658664 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277721 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419929 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551973 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574743 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361646 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457433 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361179 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533512 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216538 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.086773 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378495 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11078 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126939 s, 413 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603325 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256228 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525109 s, 998 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236279 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169678 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427388 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266607 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472697 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033874 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466152 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233331 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339746 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386753 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454949 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597856 s, 877 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311847 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0591646 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249029 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249835 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070827 s, 740 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069515 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103952 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0799076 s, 656 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638772 s, 821 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573499 s, 914 MB/s 52428800 bytes (52 MB) copied, 0.10388 s, 505 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671329 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.08025 s, 653 MB/s 52428800 bytes (52 MB) copied, 0.101563 s, 516 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0728503 s, 720 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588504 s, 891 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642896 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519719 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569378 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551774 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607201 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044212 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267513 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233481 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509131 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344569 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689214 s, 761 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472805 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738875 s, 710 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408648 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554333 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604113 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560842 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553048 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397257 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468179 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292257 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528134 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404851 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417824 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0870099 s, 603 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422781 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408728 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0890183 s, 589 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508974 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426627 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421356 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193651 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413836 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315648 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440909 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201582 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208371 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465454 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387233 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204605 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268465 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344349 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207651 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234046 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282039 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330257 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311324 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520711 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467874 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280139 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195308 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355406 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540374 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356738 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356297 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481534 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491921 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501139 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493757 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630609 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464135 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475758 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354044 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368426 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314927 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381782 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469499 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526389 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306719 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038609 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333258 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0293105 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037062 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306061 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304959 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491726 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631529 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0752314 s, 697 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174263 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721761 s, 726 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223469 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080836 s, 649 MB/s 52428800 bytes (52 MB) copied, 0.0896024 s, 585 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480723 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0325381 s, 1.6 GB/s , 0.0320355 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244657 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035269 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535494 s, 979 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662124 s, 792 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651709 s, 804 MB/s 52428800 bytes (52 MB) copied, 0.0692263 s, 757 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0399207 s, 1.3 GB/s , 0.0589802 s, 889 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0929598 s, 564 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707182 s, 741 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325445 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500869 s, 1.0 GB/s Finish ======== /mnt/lustre/f44.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x6 lmm_fid: [0x2000059f1:0x6:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x1a4:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' llite.lustre-ffff8800aadd2000.stats= snapshot_time 1746071699.958889043 secs.nsecs start_time 1746071639.301948948 secs.nsecs elapsed_time 60.656940095 secs.nsecs llite.lustre-ffff8800aae60000.stats= snapshot_time 1746071699.959049863 secs.nsecs start_time 1746071639.301974421 secs.nsecs elapsed_time 60.657075442 secs.nsecs read_bytes 1703 samples [bytes] 327680 1048576 1784676352 1870896344072192 read 151900 samples [usecs] 127 376617 113718785 1515673058281 ioctl 1 samples [reqs] open 3043 samples [usecs] 5 76402 4844455 128406800751 close 3043 samples [usecs] 5 57174 2526415 28019785267 seek 3037 samples [usecs] 2 783 15970 1012336 setattr 1 samples [usecs] 23750 23750 23750 564062500 getattr 17 samples [usecs] 129 16975 19619 288597031 mknod 1 samples [usecs] 11584 11584 11584 134189056 inode_permission 9395 samples [usecs] 1 30788 3587934 27850321606 opencount 3044 samples [reqs] 1 3043 4631447 9397203935 openclosetime 3042 samples [usecs] 10 301058 62056129 2804947053727 pcc_attach 1 samples [reqs] pcc_detach 257 samples [reqs] pcc_auto_attach 257 samples [reqs] pcc_hit_bytes 3004 samples [bytes] 52428800 52428800 157496115200 8257332324597760000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 attach 1 detach 257 auto_attach 257 llite.lustre-ffff8800aadd2000.pcc_async_threshold=1073741824 llite.lustre-ffff8800aae60000.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d44.sanity-pcc is idle now PASS 44 (66s) == sanity-pcc test 45: Concurrent read access from two mount points ========================================================== 23:55:03 (1746071703) 100+0 records in 100+0 records out 104857600 bytes (105 MB) copied, 0.330314 s, 317 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d45.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/102400 done Creating filesystem with 102400 1k blocks and 25688 inodes Filesystem UUID: 9ca937be-2d1f-4c78-b3b4-6be7680dd40c Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/13 done Writing inode tables: 0/13 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/13 done /tmp/f45.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=9ca937be-2d1f-4c78-b3b4-6be7680dd40c (extents) (64bit) (large files) (huge files) 100+0 records in 100+0 records out 104857600 bytes (105 MB) copied, 0.192016 s, 546 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f45.sanity-pcc on /mnt/pcc.d45.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) umount: /mnt/pcc.d45.sanity-pcc.2: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f45.sanity-pcc on /mnt/pcc.d45.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/102400 done Creating filesystem with 102400 1k blocks and 25688 inodes Filesystem UUID: 13c749c1-37ec-4184-8132-d5fbca8765ec Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/13 done Writing inode tables: 0/13 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/13 done /tmp/f45.sanity-pcc.2: Linux rev 1.0 ext4 filesystem data, UUID=13c749c1-37ec-4184-8132-d5fbca8765ec (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d45.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} pcc: - pccpath: /mnt/pcc.d45.sanity-pcc.2 hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800aadd2000.pcc_async_threshold=0 llite.lustre-ffff8800aae60000.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.528744 s, 99.2 MB/s /mnt/lustre/f45.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x8 lmm_fid: [0x2000059f1:0x8:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 421 0x1a5 0x240000400 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468179 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.340187 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.38134 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.411475 s, 127 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.459957 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.515618 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.507268 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.505437 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.501985 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.514333 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.50073 s, 105 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.489652 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.572396 s, 91.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.533944 s, 98.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.560609 s, 93.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594848 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.179906 s, 291 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.407324 s, 129 MB/s 52428800 bytes (52 MB) copied, 0.634222 s, 82.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376942 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200222 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0809621 s, 648 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050268 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220552 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.845132 s, 62.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.763353 s, 68.7 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.819282 s, 64.0 MB/s , 0.804456 s, 65.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.7638 s, 68.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.766266 s, 68.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.786348 s, 66.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.767218 s, 68.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.770253 s, 68.1 MB/s 52428800 bytes (52 MB) copied, 0.770533 s, 68.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.77336 s, 67.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.770699 s, 68.0 MB/s 52428800 bytes (52 MB) copied, 0.777163 s, 67.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.859124 s, 61.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.834323 s, 62.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.813088 s, 64.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0860402 s, 609 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544554 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538392 s, 974 MB/s 52428800 bytes (52 MB) copied, 0.0391478 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510778 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321736 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338747 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.113205 s, 463 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280616 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0583069 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424408 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527725 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656274 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0696009 s, 753 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634376 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711867 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0735696 s, 713 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0966507 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0825005 s, 635 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102629 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102718 s, 510 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824557 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466773 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367641 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295106 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075601 s, 693 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673797 s, 778 MB/s 52428800 bytes (52 MB) copied, 0.0346553 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526452 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479868 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378404 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198585 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493574 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0382647 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705483 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567432 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618374 s, 848 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114654 s, 457 MB/s 52428800 bytes (52 MB) copied, 0.0599401 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0773454 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107555 s, 487 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046826 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641231 s, 818 MB/s , 0.0997152 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691476 s, 758 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0947181 s, 554 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0948727 s, 553 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0887103 s, 591 MB/s 52428800 bytes (52 MB) copied, 0.112519 s, 466 MB/s , 0.0599217 s, 875 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140119 s, 374 MB/s 52428800 bytes (52 MB) copied, 0.196438 s, 267 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144354 s, 363 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131606 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139209 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132546 s, 396 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126707 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.20197 s, 260 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149553 s, 351 MB/s 52428800 bytes (52 MB) copied, 0.204663 s, 256 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.205614 s, 255 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139429 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150316 s, 349 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106147 s, 494 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12497 s, 420 MB/s 52428800 bytes (52 MB) copied, 0.107292 s, 489 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.25984 s, 202 MB/s 52428800 bytes (52 MB) copied, 0.0704104 s, 745 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129232 s, 406 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125906 s, 416 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.275576 s, 190 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647239 s, 810 MB/s 52428800 bytes (52 MB) copied, 0.12813 s, 409 MB/s , 0.09615 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109371 s, 479 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146578 s, 358 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211402 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625915 s, 838 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290029 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0849461 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279629 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165379 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469263 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498144 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490137 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441786 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027588 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303323 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370273 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345754 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339396 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774104 s, 677 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517266 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376665 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700881 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419692 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643308 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629504 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325528 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109765 s, 478 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663597 s, 790 MB/s 52428800 bytes (52 MB) copied, 0.128776 s, 407 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0874231 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0769356 s, 681 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0759852 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0829941 s, 632 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.087121 s, 602 MB/s 52428800 bytes (52 MB) copied, 0.0955406 s, 549 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518843 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778629 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.093055 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103355 s, 507 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10441 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0894873 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101084 s, 519 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471414 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112323 s, 467 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318727 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378767 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293113 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468412 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458821 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179292 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346154 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372443 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438729 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515383 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426033 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380075 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521004 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100762 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114017 s, 460 MB/s , 0.0476429 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0970719 s, 540 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0905479 s, 579 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0737731 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038804 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155462 s, 337 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826921 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224528 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106846 s, 491 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0843347 s, 622 MB/s 52428800 bytes (52 MB) copied, 0.0443941 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0839333 s, 625 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0735822 s, 713 MB/s 52428800 bytes (52 MB) copied, 0.0602407 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410195 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234841 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204725 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251528 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162821 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200169 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262621 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356851 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299127 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383617 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019784 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322867 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01727 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411931 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421161 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463057 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704883 s, 744 MB/s 52428800 bytes (52 MB) copied, 0.0776261 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0962645 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716389 s, 732 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891467 s, 588 MB/s 52428800 bytes (52 MB) copied, 0.0681755 s, 769 MB/s 52428800 bytes (52 MB) copied, 0.0517644 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505718 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0993846 s, 528 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151202 s, 347 MB/s 52428800 bytes (52 MB) copied, 0.0620142 s, 845 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598698 s, 876 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0682167 s, 769 MB/s 52428800 bytes (52 MB) copied, 0.0959955 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721065 s, 727 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119212 s, 440 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077766 s, 674 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142468 s, 368 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0720404 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14774 s, 355 MB/s 52428800 bytes (52 MB) copied, 0.0823594 s, 637 MB/s 52428800 bytes (52 MB) copied, 0.124696 s, 420 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105009 s, 499 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144777 s, 362 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685301 s, 765 MB/s , 0.0991877 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107155 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256381 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132443 s, 396 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142316 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133153 s, 394 MB/s 52428800 bytes (52 MB) copied, 0.100837 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460554 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180902 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0807105 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403967 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351167 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246773 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363809 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438607 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444961 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059114 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391279 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075958 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420984 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286045 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10078 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0956673 s, 548 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109538 s, 479 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056346 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105509 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492423 s, 1.1 GB/s , 0.0760731 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573392 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0871451 s, 602 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732552 s, 716 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045441 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0781474 s, 671 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05357 s, 979 MB/s , 0.0746056 s, 703 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583371 s, 899 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602727 s, 870 MB/s , 0.0815351 s, 643 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658942 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644851 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0921361 s, 569 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346192 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381844 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062419 s, 840 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691104 s, 759 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533093 s, 983 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.123112 s, 426 MB/s , 0.0677519 s, 774 MB/s 52428800 bytes (52 MB) copied, 0.0724673 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0759745 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868221 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879741 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11471 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824632 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0909437 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797183 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0963669 s, 544 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137499 s, 381 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102534 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127756 s, 410 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103092 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0855289 s, 613 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136473 s, 384 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0932715 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639699 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532738 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586518 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313955 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406651 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018473 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299985 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176726 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319954 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285052 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223173 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364144 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397476 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322433 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393755 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0478486 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548361 s, 956 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207304 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063003 s, 832 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200399 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579918 s, 904 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0988502 s, 530 MB/s 52428800 bytes (52 MB) copied, 0.0944799 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03852 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628712 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042756 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771093 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.101563 s, 516 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060421 s, 868 MB/s , 0.0493814 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134839 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0903798 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635711 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138135 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0737211 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497546 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526325 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612004 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134435 s, 390 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656846 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508883 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047378 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0801361 s, 654 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323929 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548128 s, 957 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114544 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0948232 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505644 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473374 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675741 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032732 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320933 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241716 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458286 s, 1.1 GB/s , 0.102902 s, 510 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810392 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393172 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495534 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332182 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496998 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637696 s, 822 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491696 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315657 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410012 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0720142 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037227 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396499 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0375821 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346029 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283913 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542647 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230921 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264466 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295867 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057675 s, 909 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272834 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025222 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415215 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500667 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0508863 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576704 s, 909 MB/s 52428800 bytes (52 MB) copied, 0.0603289 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493408 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0760386 s, 690 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616685 s, 850 MB/s 52428800 bytes (52 MB) copied, 0.0560334 s, 936 MB/s 52428800 bytes (52 MB) copied, 0.100877 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.089369 s, 587 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.179268 s, 292 MB/s 52428800 bytes (52 MB) copied, 0.169791 s, 309 MB/s 52428800 bytes (52 MB) copied, 0.112653 s, 465 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10793 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147784 s, 355 MB/s 52428800 bytes (52 MB) copied, 0.189078 s, 277 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.222849 s, 235 MB/s 52428800 bytes (52 MB) copied, 0.100034 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0820832 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15608 s, 336 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0719536 s, 729 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440341 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.103425 s, 507 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147187 s, 356 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0801257 s, 654 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0709937 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597823 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352346 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319908 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180776 s, 290 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479798 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279916 s, 1.9 GB/s , 0.0737894 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553606 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245272 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184389 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358782 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202148 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204877 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338769 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347544 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484949 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631249 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219802 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0853839 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.068385 s, 767 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440969 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.067051 s, 782 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625884 s, 838 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350787 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0999971 s, 524 MB/s 52428800 bytes (52 MB) copied, 0.0454636 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399574 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611587 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300235 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0739527 s, 709 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404663 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587848 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183165 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508633 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388933 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381933 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0916712 s, 572 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108998 s, 481 MB/s 52428800 bytes (52 MB) copied, 0.0503892 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101383 s, 517 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080829 s, 649 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396594 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551206 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334234 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310659 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0477744 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248245 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373689 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553405 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021487 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295998 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333395 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282297 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326347 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194894 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424218 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638261 s, 821 MB/s 52428800 bytes (52 MB) copied, 0.0475213 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294567 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429901 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602685 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393154 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275625 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444133 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389592 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404162 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0466953 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451952 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727153 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507204 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411652 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0801306 s, 654 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307133 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0499805 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649093 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430064 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301966 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325102 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530088 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321029 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049953 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0278694 s, 1.9 GB/s , 0.0631718 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342409 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306889 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331186 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248465 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391843 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242495 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238577 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238998 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337477 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308212 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216857 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415297 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335281 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375456 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398646 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243735 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367607 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252053 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255545 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301674 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168727 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337405 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240796 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189887 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200239 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232001 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307203 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309618 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247787 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738987 s, 709 MB/s 52428800 bytes (52 MB) copied, 0.043033 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0813881 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693759 s, 756 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767161 s, 683 MB/s 52428800 bytes (52 MB) copied, 0.0989554 s, 530 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106136 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652579 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.08195 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113798 s, 461 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0984813 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116468 s, 450 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0880847 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072027 s, 728 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101953 s, 514 MB/s 52428800 bytes (52 MB) copied, 0.136203 s, 385 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10048 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067096 s, 781 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101739 s, 515 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133522 s, 393 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444173 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0572932 s, 915 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304661 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231147 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273822 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212252 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032394 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172994 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296625 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385959 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326206 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268422 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170651 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188108 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170901 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245525 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316813 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173146 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287235 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335236 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352283 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314926 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171742 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218256 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166882 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225605 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212028 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225678 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229816 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024469 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267835 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209505 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01548 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028366 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181415 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180104 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188941 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212762 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304829 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229124 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186211 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479619 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349355 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05783 s, 907 MB/s 52428800 bytes (52 MB) copied, 0.0656381 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328853 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0670171 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061505 s, 852 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308168 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427292 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619085 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623944 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262879 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270991 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214116 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290511 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225848 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180703 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208859 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028314 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309276 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235542 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180237 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325697 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211512 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017351 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194139 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353405 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351964 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0271822 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021247 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381862 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301522 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487202 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254668 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330377 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318552 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376668 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363877 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383403 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292293 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468333 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409443 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164668 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415193 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466128 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0566902 s, 925 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.054653 s, 959 MB/s , 0.0666011 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538608 s, 973 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718592 s, 730 MB/s 52428800 bytes (52 MB) copied, 0.0431585 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040583 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721316 s, 727 MB/s 52428800 bytes (52 MB) copied, 0.0485877 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648636 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774884 s, 677 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128623 s, 408 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740328 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886328 s, 592 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534094 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077837 s, 674 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0759516 s, 690 MB/s 52428800 bytes (52 MB) copied, 0.0497977 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643298 s, 815 MB/s 52428800 bytes (52 MB) copied, 0.0459131 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320869 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110356 s, 475 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359098 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0964421 s, 544 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113063 s, 464 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501204 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.129029 s, 406 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214993 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204365 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201938 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186354 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223145 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161071 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367499 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03613 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315277 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040468 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188745 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435567 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324144 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307384 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0388755 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710849 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295832 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535013 s, 980 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269529 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517732 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716821 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358507 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528708 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374728 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345209 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295586 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508584 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327561 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238774 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319255 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389915 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493519 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509429 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572713 s, 915 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.099312 s, 528 MB/s 52428800 bytes (52 MB) copied, 0.0898588 s, 583 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109597 s, 478 MB/s 52428800 bytes (52 MB) copied, 0.122905 s, 427 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103718 s, 505 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811216 s, 646 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104176 s, 503 MB/s 52428800 bytes (52 MB) copied, 0.0787915 s, 665 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116983 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680871 s, 770 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104731 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481897 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0883692 s, 593 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109821 s, 477 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159115 s, 330 MB/s 52428800 bytes (52 MB) copied, 0.0462589 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732346 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538952 s, 973 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267413 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180435 s, 291 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181868 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169033 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017569 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182578 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0396234 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295424 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041122 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0224465 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223719 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248166 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202588 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369065 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0526974 s, 995 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056737 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0816211 s, 642 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736459 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104693 s, 501 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142813 s, 367 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572133 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155967 s, 336 MB/s 52428800 bytes (52 MB) copied, 0.10685 s, 491 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111352 s, 471 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705142 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100092 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100985 s, 519 MB/s 52428800 bytes (52 MB) copied, 0.130147 s, 403 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662094 s, 792 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103037 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12163 s, 431 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123995 s, 423 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.20502 s, 256 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120612 s, 435 MB/s 52428800 bytes (52 MB) copied, 0.0849815 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0979262 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.238371 s, 220 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11453 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578975 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117128 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108845 s, 482 MB/s 52428800 bytes (52 MB) copied, 0.0915427 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105472 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579762 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717158 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0841677 s, 623 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031216 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281581 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373847 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281129 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240063 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499941 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168335 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034091 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172565 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285464 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512085 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189721 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208767 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252033 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03031 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398014 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173253 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430183 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471618 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240592 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228797 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440673 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203796 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180857 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362151 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329329 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186872 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529232 s, 991 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285405 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234733 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0430883 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.037473 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505142 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290647 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247531 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023844 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0448489 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371338 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328859 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292775 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022392 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0644485 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435976 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521032 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427033 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443926 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249792 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456939 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433653 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509866 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447305 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434547 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490222 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0831766 s, 630 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704576 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105933 s, 495 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0885346 s, 592 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133153 s, 394 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0893045 s, 587 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132847 s, 395 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130404 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140356 s, 374 MB/s 52428800 bytes (52 MB) copied, 0.102144 s, 513 MB/s 52428800 bytes (52 MB) copied, 0.141212 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103049 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113348 s, 463 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21985 s, 238 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158329 s, 331 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149114 s, 352 MB/s 52428800 bytes (52 MB) copied, 0.110497 s, 474 MB/s 52428800 bytes (52 MB) copied, 0.110375 s, 475 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119514 s, 439 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105153 s, 499 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159996 s, 328 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102512 s, 511 MB/s 52428800 bytes (52 MB) copied, 0.14517 s, 361 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132193 s, 397 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0825452 s, 635 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142877 s, 367 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135928 s, 386 MB/s 52428800 bytes (52 MB) copied, 0.116247 s, 451 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040416 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.079922 s, 656 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442609 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695743 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301883 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025618 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220118 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025378 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222988 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257295 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373374 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291992 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221313 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732325 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067962 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391416 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448631 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231758 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415307 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402391 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586548 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035221 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379925 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424607 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630045 s, 832 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492402 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775325 s, 676 MB/s 52428800 bytes (52 MB) copied, 0.0596676 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826962 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0858623 s, 611 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122568 s, 428 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743935 s, 705 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127512 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125206 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101497 s, 517 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0862978 s, 608 MB/s 52428800 bytes (52 MB) copied, 0.0917757 s, 571 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.08173 s, 641 MB/s 52428800 bytes (52 MB) copied, 0.0806577 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142903 s, 367 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112345 s, 467 MB/s 52428800 bytes (52 MB) copied, 0.149507 s, 351 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11049 s, 475 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114014 s, 460 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.141868 s, 370 MB/s , 0.140376 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.095997 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125831 s, 417 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036923 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246486 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101002 s, 519 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226217 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203706 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732073 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725246 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583141 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158861 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021047 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325578 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342473 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322788 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310698 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256958 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029287 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188306 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220498 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033834 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255953 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220289 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302762 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163101 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198587 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0174444 s, 3.0 GB/s , 0.0174252 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271487 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292015 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232385 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280962 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171589 s, 3.1 GB/s 52428800 bytes (52 MB) copied, 0.0407402 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505411 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546979 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271507 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352426 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260432 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652497 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635084 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701987 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595461 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514039 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0690729 s, 759 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270152 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338642 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336105 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604303 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594698 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208063 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457107 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560637 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726734 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608416 s, 862 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763887 s, 686 MB/s 52428800 bytes (52 MB) copied, 0.0497242 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550498 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038966 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806511 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868972 s, 603 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0859851 s, 610 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0924175 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538296 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0790946 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0853842 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468669 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376695 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04909 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0459723 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329799 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504634 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262914 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268332 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304621 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337776 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223708 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225096 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258852 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191592 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242257 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363038 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215973 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263664 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271057 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315518 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347057 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201074 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490601 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04457 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460626 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493726 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486221 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553787 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542198 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303596 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224763 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361485 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289141 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371632 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0311027 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329801 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461704 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0476618 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373995 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435347 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440059 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585071 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236243 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599467 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191676 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247551 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619345 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430529 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0510836 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204151 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219146 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255018 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245214 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297376 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513386 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325804 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509045 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441249 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029378 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611242 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428153 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0455639 s, 1.2 GB/s , 0.0591481 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356842 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557664 s, 940 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517237 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496236 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470583 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0757181 s, 692 MB/s 52428800 bytes (52 MB) copied, 0.0779643 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673046 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0808885 s, 648 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067139 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778779 s, 673 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658575 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0969579 s, 541 MB/s 52428800 bytes (52 MB) copied, 0.0697879 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0720687 s, 727 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611699 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0846855 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0786893 s, 666 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063603 s, 824 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452954 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626394 s, 837 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279518 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344218 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200505 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492873 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656455 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190955 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235598 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216715 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021129 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270237 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225842 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0345476 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249816 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016468 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282315 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229985 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259273 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375413 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019711 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262488 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177861 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308657 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211021 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393899 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501478 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185727 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209813 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496046 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613685 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818042 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571192 s, 918 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500909 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0838417 s, 625 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672284 s, 780 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604773 s, 867 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0442195 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0588572 s, 891 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325674 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223996 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0766195 s, 684 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629489 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041709 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459751 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049543 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219391 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177539 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262672 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.021313 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.0174889 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175157 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274971 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303724 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288602 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442804 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181321 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230824 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0216155 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364353 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164256 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308896 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196431 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279235 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0293928 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184303 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232628 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420496 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163618 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186234 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230355 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487004 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482757 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556449 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050987 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078602 s, 667 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0815871 s, 643 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0942941 s, 556 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0839151 s, 625 MB/s 52428800 bytes (52 MB) copied, 0.0735631 s, 713 MB/s 52428800 bytes (52 MB) copied, 0.0939505 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100605 s, 521 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0970514 s, 540 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0827622 s, 633 MB/s 52428800 bytes (52 MB) copied, 0.0706944 s, 742 MB/s 52428800 bytes (52 MB) copied, 0.0584591 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502345 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263467 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467927 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481155 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0234988 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662178 s, 792 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10818 s, 485 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371702 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0591929 s, 886 MB/s , 0.0361185 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323669 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271335 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220583 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553601 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408355 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457787 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0739186 s, 709 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362832 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658802 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307744 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469861 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561888 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375492 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523925 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532543 s, 984 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509392 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0407275 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589117 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285991 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500618 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203419 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378665 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028209 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255013 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212253 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226172 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259332 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210619 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023966 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02384 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174831 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029242 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028894 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363531 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296745 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355744 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351625 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453311 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203476 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187979 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292613 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277095 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241234 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425738 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275332 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275588 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280213 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197578 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273724 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301638 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365495 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580147 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456249 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043469 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187168 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325036 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180446 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250472 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265012 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299028 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0341014 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426819 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513845 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057039 s, 919 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572828 s, 915 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588961 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0714074 s, 734 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604849 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579976 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544017 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058399 s, 898 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377214 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347676 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100598 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0913733 s, 574 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608137 s, 862 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027636 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254578 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541546 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233337 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181948 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342191 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179648 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210867 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273419 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205661 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270623 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316761 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332462 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02384 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281589 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228385 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320746 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291495 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530615 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327391 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516637 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0537696 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551181 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245517 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510046 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643233 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521504 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774734 s, 677 MB/s 52428800 bytes (52 MB) copied, 0.0490463 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326817 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0388536 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231511 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.026281 s, 2.0 GB/s , 0.0300624 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187716 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021711 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024434 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349353 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027687 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205106 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318538 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210536 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266466 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423379 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223146 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312555 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203793 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334067 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270071 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411827 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257813 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379508 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215129 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229512 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329499 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213911 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184019 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222557 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185062 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196055 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185083 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202679 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223399 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252484 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0375242 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396044 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357028 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0290484 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0269413 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252728 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213481 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205353 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192177 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194374 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238173 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183578 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040871 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366333 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408228 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0468857 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224859 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312469 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200939 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433432 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0632635 s, 829 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369403 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222232 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208112 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182864 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196224 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190827 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335784 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211995 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270215 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234613 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249383 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378972 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474089 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256261 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408643 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031767 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555338 s, 944 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199882 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484247 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0449635 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515291 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313327 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0240523 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259501 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054743 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193989 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184324 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183736 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263237 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213242 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264997 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272881 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218344 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241294 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329393 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310207 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193863 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274076 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019761 s, 2.7 GB/s , 0.0253754 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181232 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201656 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260955 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436204 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543848 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434269 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385132 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438395 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422731 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0698623 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463475 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0400842 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401929 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024484 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342712 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273208 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359936 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222636 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028266 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171702 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199559 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280431 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208763 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189311 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271982 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267038 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186078 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023032 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196409 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190247 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290444 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226212 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205604 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221709 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257206 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362724 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291456 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280183 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022884 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478542 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203749 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365772 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537233 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255447 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026231 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420982 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304793 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263493 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290803 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040971 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264675 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369371 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344426 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0541873 s, 968 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0371981 s, 1.4 GB/s , 0.0311193 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266851 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024322 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362574 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289678 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293793 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337768 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020444 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358156 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198175 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016832 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199454 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403704 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181527 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389814 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341777 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415831 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490134 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119566 s, 438 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0857164 s, 612 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102962 s, 509 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0722413 s, 726 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118005 s, 444 MB/s 52428800 bytes (52 MB) copied, 0.0672068 s, 780 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562645 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891006 s, 588 MB/s 52428800 bytes (52 MB) copied, 0.0814011 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680582 s, 770 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718509 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0959719 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396738 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0805637 s, 651 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742538 s, 706 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512643 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392869 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0458671 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214075 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513681 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218543 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216883 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241524 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188346 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213376 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284783 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452947 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389128 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462392 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404519 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0597303 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417822 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431073 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0474977 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748599 s, 700 MB/s 52428800 bytes (52 MB) copied, 0.0674943 s, 777 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607776 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594605 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435711 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693498 s, 756 MB/s 52428800 bytes (52 MB) copied, 0.0872829 s, 601 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810851 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634759 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420069 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0714711 s, 734 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254466 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0266902 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244449 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191193 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279975 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269585 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282771 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267652 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308258 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202229 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243456 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228953 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457924 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183339 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021857 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234777 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384106 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379953 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291145 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201018 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266778 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462494 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525779 s, 997 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288387 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185991 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212407 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247837 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254065 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245943 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246381 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325242 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336491 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0660642 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0900965 s, 582 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602874 s, 870 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0907819 s, 578 MB/s 52428800 bytes (52 MB) copied, 0.0879816 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0967081 s, 542 MB/s 52428800 bytes (52 MB) copied, 0.100536 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0959302 s, 547 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0985161 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101973 s, 514 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124565 s, 421 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105573 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566698 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110931 s, 473 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102329 s, 512 MB/s 52428800 bytes (52 MB) copied, 0.0609106 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11812 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120624 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586546 s, 894 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460867 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0517232 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453153 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127498 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109653 s, 478 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605036 s, 867 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0974909 s, 538 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443542 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.057865 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272516 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033426 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215515 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407579 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290498 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460243 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0272358 s, 1.9 GB/s , 0.0573693 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030974 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028839 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572414 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489957 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661033 s, 793 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309338 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434534 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249498 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211329 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199409 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382227 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317626 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484266 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413599 s, 1.3 GB/s , 0.0492411 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693928 s, 756 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638396 s, 821 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0894079 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0802163 s, 654 MB/s 52428800 bytes (52 MB) copied, 0.0836428 s, 627 MB/s 52428800 bytes (52 MB) copied, 0.0847845 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0981566 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0836856 s, 626 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0964767 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743941 s, 705 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104708 s, 501 MB/s 52428800 bytes (52 MB) copied, 0.0606979 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105153 s, 499 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119827 s, 438 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.076965 s, 681 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282679 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721865 s, 726 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126262 s, 415 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602344 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235957 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207838 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306494 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320012 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479362 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244254 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315999 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027242 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321334 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243712 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377612 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462552 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260931 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196759 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228704 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210265 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220577 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245425 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340502 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199291 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333518 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021392 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229147 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236375 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290853 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387351 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385265 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797657 s, 657 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063911 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508788 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0626307 s, 837 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078585 s, 667 MB/s 52428800 bytes (52 MB) copied, 0.0578077 s, 907 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105413 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111033 s, 472 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0867457 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0991769 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105436 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102149 s, 513 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847112 s, 619 MB/s 52428800 bytes (52 MB) copied, 0.11787 s, 445 MB/s 52428800 bytes (52 MB) copied, 0.0798582 s, 657 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135448 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173304 s, 303 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102138 s, 513 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.192827 s, 272 MB/s 52428800 bytes (52 MB) copied, 0.16632 s, 315 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18448 s, 284 MB/s 52428800 bytes (52 MB) copied, 0.107569 s, 487 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749606 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110709 s, 474 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.199073 s, 263 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116394 s, 450 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194445 s, 270 MB/s , 0.120176 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0978354 s, 536 MB/s 52428800 bytes (52 MB) copied, 0.155671 s, 337 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105819 s, 495 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931078 s, 563 MB/s 52428800 bytes (52 MB) copied, 0.0729922 s, 718 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0744666 s, 704 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107782 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0660163 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662903 s, 791 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612527 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0762932 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363818 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467013 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564948 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03668 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020993 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226893 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0650186 s, 806 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401772 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361167 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0334727 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336516 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324043 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331541 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306082 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427305 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444257 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628447 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110919 s, 473 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426803 s, 1.2 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400117 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.104108 s, 504 MB/s , 0.120762 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12868 s, 407 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0865505 s, 606 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0910006 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120458 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405702 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0936887 s, 560 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504123 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0750473 s, 699 MB/s , 0.0696834 s, 752 MB/s 52428800 bytes (52 MB) copied, 0.101416 s, 517 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261814 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471246 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347255 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0973392 s, 539 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04166 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479241 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033954 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054183 s, 968 MB/s 52428800 bytes (52 MB) copied, 0.0646553 s, 811 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221817 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520701 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387445 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266366 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249371 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281175 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598083 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421566 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464249 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313238 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236251 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614679 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119474 s, 439 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550272 s, 953 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287926 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0567532 s, 924 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0397739 s, 1.3 GB/s , 0.0565259 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879382 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109044 s, 481 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277184 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309279 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304624 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328976 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445533 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0488705 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030119 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347498 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030079 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276281 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264205 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372019 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296638 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261946 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023585 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436041 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263122 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252108 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041583 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473576 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0829644 s, 632 MB/s 52428800 bytes (52 MB) copied, 0.0574801 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0872083 s, 601 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109932 s, 477 MB/s 52428800 bytes (52 MB) copied, 0.124364 s, 422 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496568 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0737696 s, 711 MB/s 52428800 bytes (52 MB) copied, 0.0972593 s, 539 MB/s 52428800 bytes (52 MB) copied, 0.126724 s, 414 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557021 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120906 s, 434 MB/s 52428800 bytes (52 MB) copied, 0.0783019 s, 670 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0894048 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115602 s, 454 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480646 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525114 s, 998 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119698 s, 438 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104646 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0976097 s, 537 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705055 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518324 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621835 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0709888 s, 739 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687646 s, 762 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500261 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321553 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228577 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217926 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381086 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243017 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273655 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226331 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226594 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262128 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244989 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461873 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037928 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417683 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0454745 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349299 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576158 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047852 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0545922 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.109339 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0981597 s, 534 MB/s 52428800 bytes (52 MB) copied, 0.0709324 s, 739 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058639 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529749 s, 990 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107798 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693089 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0932116 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104105 s, 504 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130323 s, 402 MB/s , 0.144344 s, 363 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176357 s, 297 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0904399 s, 580 MB/s 52428800 bytes (52 MB) copied, 0.127554 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135499 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129877 s, 404 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139832 s, 375 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0894962 s, 586 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.120687 s, 434 MB/s , 0.18178 s, 288 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106265 s, 493 MB/s 52428800 bytes (52 MB) copied, 0.0817414 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661537 s, 793 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125777 s, 417 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229465 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677855 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189298 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260577 s, 2.0 GB/s , 0.0261169 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285115 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153623 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201019 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197368 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393541 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251667 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406044 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435931 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0712976 s, 735 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625787 s, 838 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288944 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0527746 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368616 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482681 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293672 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367926 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195808 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257019 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254001 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238905 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221031 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271138 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183362 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293816 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776109 s, 676 MB/s 52428800 bytes (52 MB) copied, 0.0787008 s, 666 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065567 s, 800 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103432 s, 507 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118984 s, 441 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112799 s, 465 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134968 s, 388 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13397 s, 391 MB/s 52428800 bytes (52 MB) copied, 0.117303 s, 447 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120139 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11126 s, 471 MB/s 52428800 bytes (52 MB) copied, 0.112915 s, 464 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151754 s, 345 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133499 s, 393 MB/s 52428800 bytes (52 MB) copied, 0.122757 s, 427 MB/s 52428800 bytes (52 MB) copied, 0.126889 s, 413 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113131 s, 463 MB/s 52428800 bytes (52 MB) copied, 0.152526 s, 344 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0926493 s, 566 MB/s 52428800 bytes (52 MB) copied, 0.124114 s, 422 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105044 s, 499 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100257 s, 523 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101934 s, 514 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129011 s, 406 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0898152 s, 584 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104094 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257221 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520374 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224204 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610949 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206179 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308205 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334132 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196135 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340851 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189565 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237809 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407216 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341006 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278318 s, 1.9 GB/s , 0.0337217 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331275 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310811 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409547 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0367401 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679148 s, 772 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0893306 s, 587 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675506 s, 776 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693825 s, 756 MB/s 52428800 bytes (52 MB) copied, 0.0519043 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627368 s, 836 MB/s 52428800 bytes (52 MB) copied, 0.0708424 s, 740 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028163 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0759122 s, 691 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0722992 s, 725 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647908 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020823 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177844 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549014 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810888 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679699 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337074 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289665 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285355 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324036 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253952 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195459 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206265 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325417 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339207 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526392 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417941 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020231 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310745 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583443 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606206 s, 865 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345935 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371775 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197542 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297371 s, 1.8 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543853 s, 964 MB/s , 0.0750028 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498445 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367068 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411684 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198118 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265802 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285554 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343395 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320938 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249404 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0357839 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454258 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646389 s, 811 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526881 s, 995 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736369 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0754465 s, 695 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666284 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466096 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0816673 s, 642 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386301 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0815872 s, 643 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104427 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241782 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475831 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.085906 s, 610 MB/s 52428800 bytes (52 MB) copied, 0.0384234 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326096 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312365 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348861 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246726 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022782 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290667 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385063 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181219 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397833 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323588 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232679 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0302733 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309368 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618386 s, 848 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440832 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0581791 s, 901 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512575 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0387325 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0903338 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549687 s, 954 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582705 s, 900 MB/s 52428800 bytes (52 MB) copied, 0.0581028 s, 902 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466442 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409367 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389513 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0918251 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417933 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0493699 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372632 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113359 s, 463 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0456945 s, 1.1 GB/s , 0.0696652 s, 753 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372045 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0640782 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603739 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103595 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0966855 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535028 s, 980 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0906095 s, 579 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240965 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108261 s, 484 MB/s 52428800 bytes (52 MB) copied, 0.0995791 s, 527 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566978 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382987 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418958 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033613 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182077 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024304 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464148 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040664 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389944 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409802 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484429 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699222 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0696438 s, 753 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569332 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0898452 s, 584 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106159 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114466 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0956286 s, 548 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0942996 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149067 s, 352 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114127 s, 459 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137395 s, 382 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148266 s, 354 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197842 s, 265 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186536 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146369 s, 358 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.128875 s, 407 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.174137 s, 301 MB/s 52428800 bytes (52 MB) copied, 0.124191 s, 422 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.228591 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.178575 s, 294 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.251096 s, 209 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150152 s, 349 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.203516 s, 258 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.14439 s, 363 MB/s , 0.174102 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.252111 s, 208 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0751025 s, 698 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824368 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065385 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155111 s, 338 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189987 s, 276 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618564 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105484 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675349 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211564 s, 248 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186419 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565426 s, 927 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422667 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0597567 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.085288 s, 615 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299052 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214267 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726948 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200356 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028517 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276654 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237946 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308778 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0402292 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315873 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217085 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275884 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439716 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350834 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272457 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410336 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033477 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289539 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208703 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523985 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651395 s, 805 MB/s 52428800 bytes (52 MB) copied, 0.0942284 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105985 s, 495 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131245 s, 399 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707945 s, 741 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119532 s, 439 MB/s 52428800 bytes (52 MB) copied, 0.0997537 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067515 s, 777 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126581 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0856954 s, 612 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100843 s, 520 MB/s 52428800 bytes (52 MB) copied, 0.123943 s, 423 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0910421 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119098 s, 440 MB/s 52428800 bytes (52 MB) copied, 0.0553356 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0952735 s, 550 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181019 s, 290 MB/s 52428800 bytes (52 MB) copied, 0.082656 s, 634 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498873 s, 1.1 GB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0691467 s, 758 MB/s , 0.0423067 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049806 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13838 s, 379 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035502 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0716522 s, 732 MB/s , 0.0408052 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223059 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293059 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260165 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030621 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0191601 s, 2.7 GB/s , 0.0329447 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318465 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0230067 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0424277 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269994 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338413 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513263 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345813 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393785 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538145 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0838599 s, 625 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708905 s, 740 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369481 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548655 s, 956 MB/s 52428800 bytes (52 MB) copied, 0.0721904 s, 726 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630942 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397622 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228382 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370861 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283499 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264267 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338202 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243468 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274511 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402114 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304401 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412588 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261066 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025973 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235332 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305014 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018164 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484987 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329012 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0470435 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335732 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484271 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446971 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708743 s, 740 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707982 s, 741 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0922121 s, 569 MB/s 52428800 bytes (52 MB) copied, 0.0939619 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107164 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619546 s, 846 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0869115 s, 603 MB/s 52428800 bytes (52 MB) copied, 0.0937043 s, 560 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113248 s, 463 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067552 s, 776 MB/s 52428800 bytes (52 MB) copied, 0.0526984 s, 995 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105274 s, 498 MB/s 52428800 bytes (52 MB) copied, 0.0854182 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392731 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486554 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0870814 s, 602 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042615 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.091794 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328844 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183975 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0839265 s, 625 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781983 s, 670 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230788 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064078 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229235 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228301 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378542 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315128 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320211 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472232 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311243 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473711 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028523 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363245 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044627 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527423 s, 994 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561659 s, 933 MB/s 52428800 bytes (52 MB) copied, 0.0454198 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368184 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427054 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475473 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059888 s, 875 MB/s 52428800 bytes (52 MB) copied, 0.0700997 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0853823 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018012 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378835 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304542 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290297 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193136 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038807 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616428 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379253 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244905 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337555 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210797 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362612 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0341133 s, 1.5 GB/s , 0.0289727 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193763 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188436 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0288543 s, 1.8 GB/s , 0.0247242 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209243 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243185 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440952 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030174 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422199 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048075 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0532677 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025215 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573222 s, 915 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322352 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0869664 s, 603 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0692962 s, 757 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646551 s, 811 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0804543 s, 652 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122742 s, 427 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743751 s, 705 MB/s 52428800 bytes (52 MB) copied, 0.108563 s, 483 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701124 s, 748 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743602 s, 705 MB/s 52428800 bytes (52 MB) copied, 0.0924971 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0974253 s, 538 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144974 s, 362 MB/s 52428800 bytes (52 MB) copied, 0.087415 s, 600 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104281 s, 503 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101327 s, 517 MB/s 52428800 bytes (52 MB) copied, 0.0590376 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.073155 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0922505 s, 568 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685333 s, 765 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539842 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0917204 s, 572 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080167 s, 654 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582997 s, 899 MB/s 52428800 bytes (52 MB) copied, 0.0295663 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278345 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329931 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434277 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339837 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268096 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287067 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024085 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316499 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041191 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283154 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253961 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194298 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341586 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0963874 s, 544 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0866419 s, 605 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756296 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048148 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.102705 s, 510 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119914 s, 437 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.15776 s, 332 MB/s , 0.0724327 s, 724 MB/s 52428800 bytes (52 MB) copied, 0.0863069 s, 607 MB/s 52428800 bytes (52 MB) copied, 0.0948659 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0937212 s, 559 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0971918 s, 539 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0966732 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0820347 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514594 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105889 s, 495 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583981 s, 898 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10243 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0960954 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487888 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146157 s, 359 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460685 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489703 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110715 s, 474 MB/s 52428800 bytes (52 MB) copied, 0.0502687 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513359 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181164 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018985 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304146 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183858 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207364 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042854 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020974 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343446 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196283 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411379 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357455 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05081 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741214 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.08959 s, 585 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0892856 s, 587 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124266 s, 422 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0860938 s, 609 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111876 s, 469 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742147 s, 706 MB/s 52428800 bytes (52 MB) copied, 0.0924237 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0940083 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0999807 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850178 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701328 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847737 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588922 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296693 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276921 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100079 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107755 s, 487 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388065 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172928 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175779 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251398 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365724 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236212 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590425 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437165 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699179 s, 750 MB/s 52428800 bytes (52 MB) copied, 0.0551736 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390462 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694677 s, 755 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033387 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0650921 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0768581 s, 682 MB/s 52428800 bytes (52 MB) copied, 0.0472932 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588089 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297589 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178907 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228194 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358172 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209596 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222434 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029746 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262406 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254956 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0636732 s, 823 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559268 s, 937 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748541 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482887 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710564 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715353 s, 733 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106018 s, 495 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0979823 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634373 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127245 s, 412 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679241 s, 772 MB/s 52428800 bytes (52 MB) copied, 0.065691 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638673 s, 821 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471336 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653342 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.094941 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0893936 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484972 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542599 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13444 s, 390 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.081661 s, 642 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077046 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017984 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196142 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275723 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293005 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197746 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032085 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265167 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208484 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227841 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258952 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293021 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396715 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299177 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307603 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0351063 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031724 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275201 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395023 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293545 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02887 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819196 s, 640 MB/s 52428800 bytes (52 MB) copied, 0.066482 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049842 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667236 s, 786 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572806 s, 915 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495481 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0793893 s, 660 MB/s 52428800 bytes (52 MB) copied, 0.0596091 s, 880 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431778 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613798 s, 854 MB/s 52428800 bytes (52 MB) copied, 0.0901092 s, 582 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149324 s, 351 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157917 s, 332 MB/s 52428800 bytes (52 MB) copied, 0.195816 s, 268 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146661 s, 357 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107821 s, 486 MB/s 52428800 bytes (52 MB) copied, 0.139485 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130664 s, 401 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120197 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112963 s, 464 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0935255 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0892291 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129406 s, 405 MB/s , 0.171491 s, 306 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143018 s, 367 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181511 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125403 s, 418 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140542 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107778 s, 486 MB/s 52428800 bytes (52 MB) copied, 0.124446 s, 421 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0712754 s, 736 MB/s 52428800 bytes (52 MB) copied, 0.0855603 s, 613 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114524 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683248 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749287 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128152 s, 409 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677114 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0985999 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614997 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503014 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249987 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177722 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193803 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0299874 s, 1.7 GB/s , 0.026829 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262744 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242937 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249381 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190011 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219083 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021233 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212413 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244747 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406036 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446524 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218274 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370266 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038022 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294926 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554588 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580265 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475618 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593109 s, 884 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056654 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504517 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0350828 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691806 s, 758 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306717 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520654 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0759957 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517549 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767928 s, 683 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495427 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0571277 s, 918 MB/s 52428800 bytes (52 MB) copied, 0.0344404 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422015 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299425 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182626 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413671 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411432 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310408 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306307 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300356 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280011 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213676 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260756 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211664 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246507 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268807 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210806 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263336 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509772 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491413 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251704 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382061 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411664 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201653 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294961 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385225 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180367 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327565 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048881 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0681701 s, 769 MB/s 52428800 bytes (52 MB) copied, 0.071089 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564009 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0783615 s, 669 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726303 s, 722 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109513 s, 479 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0974263 s, 538 MB/s , 0.0750529 s, 699 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879371 s, 596 MB/s 52428800 bytes (52 MB) copied, 0.0760148 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101787 s, 515 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891182 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711855 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106067 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10718 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134269 s, 390 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0837533 s, 626 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0760992 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457718 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580471 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424959 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478753 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109706 s, 478 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774839 s, 677 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251333 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452606 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219808 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273024 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208293 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399438 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0698408 s, 751 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0975561 s, 537 MB/s 52428800 bytes (52 MB) copied, 0.0805409 s, 651 MB/s 52428800 bytes (52 MB) copied, 0.0801656 s, 654 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597015 s, 878 MB/s 52428800 bytes (52 MB) copied, 0.0970896 s, 540 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794915 s, 660 MB/s 52428800 bytes (52 MB) copied, 0.0544385 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126415 s, 415 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320593 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115893 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0962486 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.084981 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0813847 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238499 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182325 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183789 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255705 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263708 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047958 s, 1.1 GB/s 50+0 records in 50+0 records out , 0.0646933 s, 810 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0492173 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0860031 s, 610 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471152 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.050332 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706405 s, 742 MB/s 52428800 bytes (52 MB) copied, 0.0983688 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535828 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0712989 s, 735 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879971 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507151 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685139 s, 765 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0650336 s, 806 MB/s 52428800 bytes (52 MB) copied, 0.0680732 s, 770 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470387 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.053082 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398118 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026011 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264882 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361028 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432448 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032301 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556767 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359664 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348666 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215292 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257009 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223736 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236934 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0636899 s, 823 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194967 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280632 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231973 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189189 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205427 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046048 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405608 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183748 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492967 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476262 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416254 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225582 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400189 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201073 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02407 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321276 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414295 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564997 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236078 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195899 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554049 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456555 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383127 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715719 s, 733 MB/s 52428800 bytes (52 MB) copied, 0.0820662 s, 639 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03982 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060571 s, 866 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464022 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366564 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0684574 s, 766 MB/s 52428800 bytes (52 MB) copied, 0.0716659 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296193 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246774 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170759 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407492 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306346 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580917 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599094 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309081 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677078 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0828459 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0804135 s, 652 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706751 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10695 s, 490 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102303 s, 512 MB/s 52428800 bytes (52 MB) copied, 0.0604129 s, 868 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0938397 s, 559 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891547 s, 588 MB/s 52428800 bytes (52 MB) copied, 0.129843 s, 404 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112475 s, 466 MB/s 52428800 bytes (52 MB) copied, 0.0941693 s, 557 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781068 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0719652 s, 729 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534267 s, 981 MB/s 52428800 bytes (52 MB) copied, 0.161251 s, 325 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663343 s, 790 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054477 s, 962 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110856 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0848422 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0964871 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0958746 s, 547 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0751939 s, 697 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210233 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035702 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051801 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341468 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431679 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213038 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440152 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318825 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300652 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564848 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028236 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310069 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432454 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271519 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177066 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171879 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423119 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231304 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288033 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324543 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511963 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465404 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354557 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493202 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0470364 s, 1.1 GB/s , 0.0559687 s, 937 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506852 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452311 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224166 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0822781 s, 637 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0829642 s, 632 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0682861 s, 768 MB/s 52428800 bytes (52 MB) copied, 0.0865458 s, 606 MB/s 52428800 bytes (52 MB) copied, 0.0837144 s, 626 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0674228 s, 778 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0795867 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600403 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665068 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554919 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555983 s, 943 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540374 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677986 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0821438 s, 638 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028919 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558459 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229514 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261351 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219978 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306665 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267535 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403324 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287527 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327855 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176832 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298571 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210655 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432006 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243613 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326756 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174999 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278531 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286381 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294235 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374372 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511293 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328139 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288401 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206263 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581537 s, 902 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334203 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327785 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213211 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198691 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280925 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213652 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044624 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0492263 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0497081 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0782712 s, 670 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0912757 s, 574 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0875872 s, 599 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0855749 s, 613 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119949 s, 437 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0773044 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106096 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.095562 s, 549 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115374 s, 454 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100955 s, 519 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0770651 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111261 s, 471 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12925 s, 406 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144448 s, 363 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101743 s, 515 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111481 s, 470 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774601 s, 677 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0768432 s, 682 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711941 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0980283 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794915 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695084 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339385 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589887 s, 889 MB/s 52428800 bytes (52 MB) copied, 0.0486945 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233192 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198776 s, 2.6 GB/s , 0.0246486 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205638 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251276 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206325 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244946 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189656 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260285 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396592 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305506 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228045 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202282 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206602 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182579 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263217 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020772 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230789 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316412 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242691 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256778 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228251 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191326 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260004 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031895 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257962 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296961 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241952 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312101 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417978 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252838 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344279 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253884 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208467 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318905 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314366 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200752 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405112 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220087 s, 2.4 GB/s , 0.0244966 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030436 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025346 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191261 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281832 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238662 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041035 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213167 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291595 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0290252 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541314 s, 969 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019603 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637873 s, 822 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459868 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624856 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549903 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04046 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042927 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377249 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590002 s, 889 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365356 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332042 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408253 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511116 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315686 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0724875 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484177 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641733 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0936937 s, 560 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606077 s, 865 MB/s 52428800 bytes (52 MB) copied, 0.0573779 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0966628 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776031 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039816 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433566 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411601 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0861832 s, 608 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211044 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633339 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0792974 s, 661 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.099343 s, 528 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.085445 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183206 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210106 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194168 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298879 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269778 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267815 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257064 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216981 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202943 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266998 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277321 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470773 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408606 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028609 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264404 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596208 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198624 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345644 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212183 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590278 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504339 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493055 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.084905 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824792 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0720068 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774446 s, 677 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711092 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0928387 s, 565 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622934 s, 842 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0807988 s, 649 MB/s , 0.0980943 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672896 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613589 s, 854 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0855845 s, 613 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.098947 s, 530 MB/s 52428800 bytes (52 MB) copied, 0.0690601 s, 759 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0769045 s, 682 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600447 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0788524 s, 665 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0846384 s, 619 MB/s 52428800 bytes (52 MB) copied, 0.0857069 s, 612 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606502 s, 864 MB/s 52428800 bytes (52 MB) copied, 0.0701464 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489539 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503737 s, 1.0 GB/s , 0.0468453 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026488 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379004 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030881 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271835 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187157 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320951 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357885 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321198 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027537 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219576 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452873 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205919 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224836 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262611 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379407 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528798 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261497 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336978 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285325 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497016 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306746 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394485 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435186 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158738 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438032 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339974 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026542 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169981 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332788 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383075 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303936 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186963 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494358 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029646 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590627 s, 888 MB/s , 0.0407319 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218452 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0382721 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584268 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251262 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262727 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404801 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213327 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416907 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372077 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298329 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268489 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262856 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340357 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633597 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318328 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264296 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263137 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247639 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224971 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195918 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266454 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026932 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301552 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411652 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028693 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323558 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264356 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310393 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03127 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246065 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330169 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024391 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240211 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234105 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187981 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159062 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364784 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029649 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249505 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301436 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208733 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300605 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236041 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373588 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342437 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453565 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522404 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449243 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0570136 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507382 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767638 s, 683 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0799742 s, 656 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675164 s, 777 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046126 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721397 s, 727 MB/s 52428800 bytes (52 MB) copied, 0.0751424 s, 698 MB/s 52428800 bytes (52 MB) copied, 0.100028 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794405 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395122 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0863057 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562988 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513515 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263782 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407709 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552828 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222722 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315092 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477941 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178388 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639325 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226343 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021655 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022955 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465101 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266218 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201682 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270184 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597298 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037234 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474528 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375264 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028403 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024278 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399164 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628236 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285254 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187778 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259659 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200758 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180032 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187352 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179561 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189517 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021362 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235498 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321479 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478645 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300346 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03073 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229137 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515708 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427631 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0349215 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371793 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034298 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508328 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249713 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475424 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410806 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035839 s, 1.5 GB/s , 0.0398684 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236753 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256648 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329016 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218148 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0883095 s, 594 MB/s 52428800 bytes (52 MB) copied, 0.0796473 s, 658 MB/s 52428800 bytes (52 MB) copied, 0.130555 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818662 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103994 s, 504 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113802 s, 461 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115947 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.111281 s, 471 MB/s , 0.121343 s, 432 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176878 s, 296 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662016 s, 792 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125024 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127345 s, 412 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560263 s, 936 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10443 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0996203 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501988 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107428 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0645799 s, 812 MB/s 52428800 bytes (52 MB) copied, 0.134427 s, 390 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0777617 s, 674 MB/s , 0.12592 s, 416 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486926 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224449 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0939883 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391847 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426027 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180286 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416069 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171936 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017394 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219838 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186918 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209683 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188914 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186503 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323931 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290641 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231468 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211784 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266985 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465118 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312514 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368063 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279065 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530351 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270284 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284193 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416815 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362045 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221434 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041453 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508128 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037311 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400674 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460732 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.083091 s, 631 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580645 s, 903 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661342 s, 793 MB/s 52428800 bytes (52 MB) copied, 0.0513321 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789431 s, 664 MB/s 52428800 bytes (52 MB) copied, 0.0903452 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655862 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036837 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0870748 s, 602 MB/s 52428800 bytes (52 MB) copied, 0.0564544 s, 929 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886347 s, 592 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323992 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0561412 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695944 s, 753 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11221 s, 467 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781009 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059679 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515462 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0530566 s, 988 MB/s , 0.0526491 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058792 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505842 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584347 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472169 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310688 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0348225 s, 1.5 GB/s , 0.0229945 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178545 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023748 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274496 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258971 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262344 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279473 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220838 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300534 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288795 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280347 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333249 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315829 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277874 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205331 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205609 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439146 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239077 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276059 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240635 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201829 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282695 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329288 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250085 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326272 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216782 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310417 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0238505 s, 2.2 GB/s , 0.0178209 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221153 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316104 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450912 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051724 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397708 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050698 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.041126 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221081 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236571 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553819 s, 947 MB/s 52428800 bytes (52 MB) copied, 0.046642 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625175 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397276 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741261 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204912 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448374 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218149 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022634 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411317 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533549 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050926 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441559 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022602 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0952355 s, 551 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064788 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104779 s, 500 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461003 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933668 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0730176 s, 718 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.204842 s, 256 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121735 s, 431 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100567 s, 521 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.212512 s, 247 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134388 s, 390 MB/s , 0.222957 s, 235 MB/s 52428800 bytes (52 MB) copied, 0.253233 s, 207 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879173 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.169147 s, 310 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111848 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136875 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130814 s, 401 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150608 s, 348 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141418 s, 371 MB/s 52428800 bytes (52 MB) copied, 0.156109 s, 336 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142526 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113738 s, 461 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741622 s, 707 MB/s 52428800 bytes (52 MB) copied, 0.0900407 s, 582 MB/s 52428800 bytes (52 MB) copied, 0.0926033 s, 566 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108975 s, 481 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529125 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259011 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451654 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131563 s, 399 MB/s 52428800 bytes (52 MB) copied, 0.136755 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710499 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061389 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237456 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329258 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305737 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461293 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381875 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276354 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342555 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0274647 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0915247 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602291 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366699 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462826 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381412 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0962986 s, 544 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614624 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0992864 s, 528 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879395 s, 596 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137289 s, 382 MB/s 52428800 bytes (52 MB) copied, 0.0884265 s, 593 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850249 s, 617 MB/s 52428800 bytes (52 MB) copied, 0.103262 s, 508 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147069 s, 356 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120237 s, 436 MB/s 52428800 bytes (52 MB) copied, 0.0883736 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.095959 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824081 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0945868 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0839801 s, 624 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0981197 s, 534 MB/s , 0.158869 s, 330 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704929 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107454 s, 488 MB/s 52428800 bytes (52 MB) copied, 0.0973367 s, 539 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054552 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141129 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184032 s, 285 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519573 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419071 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694943 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140447 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355337 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237848 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210707 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209762 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.021106 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472595 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341421 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607163 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427055 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623573 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666215 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0937413 s, 559 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614792 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593104 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0878434 s, 597 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818285 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320972 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545289 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435318 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404381 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.081751 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048942 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0713706 s, 735 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665758 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043501 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102043 s, 514 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404932 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202016 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0827169 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0785214 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248649 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289362 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029047 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343119 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302061 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318855 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167196 s, 3.1 GB/s 52428800 bytes (52 MB) copied, 0.017463 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291922 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178634 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202942 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426447 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340594 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327159 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432131 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428931 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810599 s, 647 MB/s 52428800 bytes (52 MB) copied, 0.0397635 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0925436 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507886 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395176 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695742 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135024 s, 388 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.082301 s, 637 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101359 s, 517 MB/s 52428800 bytes (52 MB) copied, 0.203608 s, 257 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118003 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139867 s, 375 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102596 s, 511 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.160842 s, 326 MB/s 50+0 records in 50+0 records out , 0.240512 s, 218 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141914 s, 369 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.226492 s, 231 MB/s 52428800 bytes (52 MB) copied, 0.1767 s, 297 MB/s 52428800 bytes (52 MB) copied, 0.175671 s, 298 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21459 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174248 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.208163 s, 252 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136939 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180279 s, 291 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164054 s, 320 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.230488 s, 227 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.215834 s, 243 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166229 s, 315 MB/s 52428800 bytes (52 MB) copied, 0.197284 s, 266 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353593 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167096 s, 314 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0940528 s, 557 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850069 s, 617 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119336 s, 439 MB/s 52428800 bytes (52 MB) copied, 0.292221 s, 179 MB/s 52428800 bytes (52 MB) copied, 0.0546609 s, 959 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225311 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0791529 s, 662 MB/s , 0.0596498 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134722 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125537 s, 418 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502759 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643074 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587562 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455744 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190909 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399653 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187589 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303482 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492356 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0865303 s, 606 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555751 s, 943 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414091 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500538 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059466 s, 882 MB/s 52428800 bytes (52 MB) copied, 0.0842363 s, 622 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104389 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749981 s, 699 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0813435 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0911537 s, 575 MB/s 52428800 bytes (52 MB) copied, 0.0851649 s, 616 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064313 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114432 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103938 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132879 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626137 s, 837 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0936047 s, 560 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.141006 s, 372 MB/s , 0.0806335 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756498 s, 693 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127057 s, 413 MB/s 52428800 bytes (52 MB) copied, 0.107562 s, 487 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0974481 s, 538 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106044 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120049 s, 437 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151193 s, 347 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123491 s, 425 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11296 s, 464 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11771 s, 445 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111398 s, 471 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115748 s, 453 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177481 s, 295 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0431508 s, 1.2 GB/s , 0.0919629 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329846 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133224 s, 394 MB/s 52428800 bytes (52 MB) copied, 0.0819548 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241575 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195067 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265205 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317859 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033506 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469593 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037515 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222005 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403463 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282749 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207966 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285595 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026625 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413649 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305297 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020045 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329725 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426942 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383449 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0351056 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196861 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373586 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0297548 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035789 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264173 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333339 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054631 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0864091 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523823 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463053 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777267 s, 675 MB/s 52428800 bytes (52 MB) copied, 0.0846175 s, 620 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385844 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0658547 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390251 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260572 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.133072 s, 394 MB/s , 0.075191 s, 697 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147415 s, 356 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0743414 s, 705 MB/s , 0.116552 s, 450 MB/s 52428800 bytes (52 MB) copied, 0.152871 s, 343 MB/s 52428800 bytes (52 MB) copied, 0.120537 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104092 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0976036 s, 537 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0768206 s, 682 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0906177 s, 579 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.169859 s, 309 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580341 s, 903 MB/s 52428800 bytes (52 MB) copied, 0.0543099 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130693 s, 401 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444356 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355926 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666518 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667082 s, 786 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433285 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050482 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243444 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203908 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265381 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172643 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031372 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203023 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306087 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213018 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404224 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469052 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384555 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240096 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427611 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271312 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253075 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230587 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404623 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181599 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269722 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261618 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019863 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537536 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448522 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606223 s, 865 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0870203 s, 602 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.103385 s, 507 MB/s 52428800 bytes (52 MB) copied, 0.148851 s, 352 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603288 s, 869 MB/s 52428800 bytes (52 MB) copied, 0.109855 s, 477 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202029 s, 260 MB/s 52428800 bytes (52 MB) copied, 0.117356 s, 447 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879752 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145366 s, 361 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.190511 s, 275 MB/s 52428800 bytes (52 MB) copied, 0.173833 s, 302 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436122 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134307 s, 390 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130043 s, 403 MB/s 52428800 bytes (52 MB) copied, 0.178961 s, 293 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0968017 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0939185 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0783367 s, 669 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717355 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708483 s, 740 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0735129 s, 713 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408366 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281379 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0893108 s, 587 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164187 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338902 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275938 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243983 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292077 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243833 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191258 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237681 s, 2.2 GB/s , 0.0346606 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017658 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346639 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339474 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196087 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202576 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313879 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020721 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157546 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397409 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344319 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337141 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017673 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611899 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332021 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320787 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480117 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331823 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506368 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530322 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321193 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305864 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592869 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520104 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672005 s, 780 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0927218 s, 565 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0857419 s, 611 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0795955 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075857 s, 691 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524954 s, 999 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0974562 s, 538 MB/s 52428800 bytes (52 MB) copied, 0.12527 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667462 s, 785 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100958 s, 519 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026472 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0938102 s, 559 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198916 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062023 s, 845 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0831188 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392026 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235834 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390806 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253401 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282631 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420367 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172687 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045808 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030967 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336472 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703915 s, 745 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402921 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594207 s, 882 MB/s 52428800 bytes (52 MB) copied, 0.0431569 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622644 s, 842 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120345 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477365 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.11349 s, 462 MB/s , 0.0810527 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416733 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296278 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186103 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035294 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306661 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178286 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310467 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237093 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388802 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172155 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457054 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259261 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285548 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284811 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273297 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394618 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252504 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031227 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418428 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552737 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513278 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634025 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563015 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688988 s, 761 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418555 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109592 s, 478 MB/s 52428800 bytes (52 MB) copied, 0.0607127 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107335 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0570089 s, 920 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463514 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348626 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.070341 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394281 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226154 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372194 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386946 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026428 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220863 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023663 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193248 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319116 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039126 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074996 s, 699 MB/s 52428800 bytes (52 MB) copied, 0.0902144 s, 581 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614132 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114305 s, 459 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703831 s, 745 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109744 s, 478 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100559 s, 521 MB/s 52428800 bytes (52 MB) copied, 0.110015 s, 477 MB/s 52428800 bytes (52 MB) copied, 0.088224 s, 594 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0782554 s, 670 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1065 s, 492 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107191 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110534 s, 474 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756356 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624674 s, 839 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0860317 s, 609 MB/s 52428800 bytes (52 MB) copied, 0.0627747 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281871 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0852156 s, 615 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0881476 s, 595 MB/s , 0.0444331 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789625 s, 664 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606896 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181398 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327799 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032348 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235035 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163556 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242082 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167131 s, 3.1 GB/s 52428800 bytes (52 MB) copied, 0.0239202 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254623 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175775 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0178906 s, 2.9 GB/s , 0.0231892 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032618 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301506 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266511 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329469 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251455 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267013 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264328 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416079 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355439 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339581 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248876 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167457 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192576 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194707 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279153 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218942 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276743 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189181 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201633 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192826 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168243 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186607 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165004 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226775 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383333 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413809 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421481 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522957 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0712283 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373073 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533065 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554232 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343452 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625175 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254725 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640243 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370222 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479085 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0858998 s, 610 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322246 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210772 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238736 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016716 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202559 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193805 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255905 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027834 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193146 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225829 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253195 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264033 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170115 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018629 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167079 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199009 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187692 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223863 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234157 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032579 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316387 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289324 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186672 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202318 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235111 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312644 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270921 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016981 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232312 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274078 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0524214 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287488 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286221 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405119 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0384468 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307882 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304337 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221442 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185546 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241071 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205559 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018514 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184304 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220323 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028607 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197518 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213316 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226212 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294131 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266247 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397321 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039797 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202887 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254392 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034676 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353961 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191549 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200779 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017325 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197542 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031173 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270436 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303102 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306519 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508694 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348415 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430848 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332164 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182202 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456669 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310076 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0821302 s, 638 MB/s 52428800 bytes (52 MB) copied, 0.0711548 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261295 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062719 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478641 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024444 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0208231 s, 2.5 GB/s , 0.0187096 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191178 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173684 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228384 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255833 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358097 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273801 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154939 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232599 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208955 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358576 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032296 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0226127 s, 2.3 GB/s , 0.0315951 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190781 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315124 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294563 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680535 s, 770 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543456 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708285 s, 740 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664985 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687172 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634487 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0803573 s, 652 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0788123 s, 665 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0737966 s, 710 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683103 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109127 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109357 s, 479 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110236 s, 476 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107525 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0760379 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420192 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0877124 s, 598 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116022 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758763 s, 691 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509292 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0543967 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366921 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366978 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0951628 s, 551 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117555 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0953508 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397926 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212606 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227252 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174241 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192854 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217068 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303148 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618646 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166859 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026276 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0849463 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196141 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332257 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472824 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179809 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262779 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198354 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192877 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163652 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166344 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186847 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263129 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273399 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318944 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406986 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309052 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428536 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228565 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337153 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449311 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311059 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473511 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331826 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220135 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419376 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472566 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037763 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438523 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545522 s, 961 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732262 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806172 s, 650 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701744 s, 747 MB/s 52428800 bytes (52 MB) copied, 0.0841183 s, 623 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0823912 s, 636 MB/s 52428800 bytes (52 MB) copied, 0.0682361 s, 768 MB/s 52428800 bytes (52 MB) copied, 0.115578 s, 454 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378234 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.11521 s, 455 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686674 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143067 s, 366 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550896 s, 952 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.07368 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134254 s, 391 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.076921 s, 682 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100772 s, 520 MB/s 52428800 bytes (52 MB) copied, 0.0474687 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11464 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716929 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072101 s, 727 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053932 s, 972 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517106 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0670952 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571266 s, 918 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426998 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187304 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214434 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262286 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381019 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181312 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232536 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324183 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237509 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183657 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185163 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250473 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223587 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231571 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216455 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264807 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186831 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218982 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239478 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432731 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335087 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311965 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515465 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369645 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237386 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02874 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168261 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334329 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181523 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360822 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275426 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353558 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763543 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291632 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0719774 s, 728 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481941 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774939 s, 677 MB/s 52428800 bytes (52 MB) copied, 0.079556 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043411 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039514 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0913315 s, 574 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397916 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536328 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0830237 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342673 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425295 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330809 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326322 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316505 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468222 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0338371 s, 1.5 GB/s , 0.0506722 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032594 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298405 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280791 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308075 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246568 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346092 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024046 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281525 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223982 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180315 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0375879 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416257 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227301 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508716 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285929 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225645 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535995 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480285 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478152 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0463791 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763422 s, 687 MB/s 52428800 bytes (52 MB) copied, 0.0629468 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596195 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706976 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530604 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492365 s, 1.1 GB/s , 0.042786 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691171 s, 759 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603517 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301742 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396698 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247676 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312181 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234742 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173031 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192553 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367782 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255537 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229003 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211762 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270833 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215943 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263583 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364388 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351508 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310584 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492177 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231325 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328499 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348161 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0265497 s, 2.0 GB/s , 0.0408204 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348758 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320311 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189745 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212171 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246811 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408134 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202844 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373542 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0592195 s, 885 MB/s , 0.0177051 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316294 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318547 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316527 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208982 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306457 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0515048 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049504 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245251 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346317 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623225 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377335 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0921608 s, 569 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368973 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383062 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625193 s, 839 MB/s 52428800 bytes (52 MB) copied, 0.0514989 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588625 s, 891 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199993 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635315 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0817326 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710451 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232301 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345315 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355162 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202708 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652103 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174936 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383024 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520042 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028381 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291926 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236653 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367427 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310812 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397098 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287047 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059346 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659982 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686981 s, 763 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758701 s, 691 MB/s 52428800 bytes (52 MB) copied, 0.0693413 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0956098 s, 548 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465673 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0947405 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727611 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585943 s, 895 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138498 s, 379 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576934 s, 909 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12358 s, 424 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462527 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0751213 s, 698 MB/s , 0.0248949 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.12623 s, 415 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406845 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181442 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302634 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230833 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293188 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324607 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221871 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223411 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219595 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411643 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427037 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0266684 s, 2.0 GB/s , 0.0279602 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330607 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196538 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281993 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382614 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207761 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165137 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451047 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491489 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200392 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0562876 s, 931 MB/s , 0.0429738 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382454 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740327 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054434 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0839586 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589633 s, 889 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0713503 s, 735 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118232 s, 443 MB/s 52428800 bytes (52 MB) copied, 0.103288 s, 508 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0833106 s, 629 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406482 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826909 s, 634 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070721 s, 741 MB/s 52428800 bytes (52 MB) copied, 0.0377594 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580224 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742363 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0786417 s, 667 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286488 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0490947 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195924 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454827 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266028 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299912 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267625 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367445 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234671 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0344262 s, 1.5 GB/s , 0.0350112 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211883 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213882 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478597 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182825 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283483 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042321 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033584 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283318 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524836 s, 999 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302084 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464757 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216253 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262089 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271927 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279713 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248572 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316797 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268194 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454414 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341214 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614617 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704285 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503668 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582584 s, 900 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0868909 s, 603 MB/s , 0.048201 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538507 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520845 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643191 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675571 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571938 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0923252 s, 568 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246878 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345529 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396236 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219153 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228629 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224739 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021539 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296741 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402758 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474258 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0391604 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640204 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217458 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263714 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0661615 s, 792 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484896 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0385149 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0506331 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0397837 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.077097 s, 680 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546817 s, 959 MB/s 52428800 bytes (52 MB) copied, 0.0429302 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383727 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432516 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680241 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549273 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360588 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0573343 s, 914 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332533 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0479764 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216008 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203377 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170897 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175453 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180399 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212203 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300987 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206893 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347567 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226601 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295601 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219502 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180914 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220625 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188114 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214496 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187153 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024509 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194949 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289102 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247986 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230805 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030338 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183431 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460749 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04116 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387515 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02432 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223984 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363125 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257409 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249687 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334586 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198899 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275756 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569746 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586313 s, 894 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067783 s, 773 MB/s 52428800 bytes (52 MB) copied, 0.0686891 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677384 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0787647 s, 666 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0674353 s, 777 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0964601 s, 544 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0982201 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0854552 s, 614 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0615367 s, 852 MB/s , 0.0676045 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.085252 s, 615 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649877 s, 807 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743942 s, 705 MB/s 52428800 bytes (52 MB) copied, 0.077203 s, 679 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333192 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0698884 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667553 s, 785 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868175 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208142 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278195 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192362 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284381 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362371 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291941 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448018 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378507 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679248 s, 772 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0739192 s, 709 MB/s 52428800 bytes (52 MB) copied, 0.0649788 s, 807 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624038 s, 840 MB/s 52428800 bytes (52 MB) copied, 0.0732378 s, 716 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731251 s, 717 MB/s 52428800 bytes (52 MB) copied, 0.0736788 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114714 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0952645 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879606 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105333 s, 498 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619178 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879497 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775182 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372856 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382471 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103978 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127973 s, 410 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147205 s, 356 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118055 s, 444 MB/s , 0.0976624 s, 537 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.088888 s, 590 MB/s 52428800 bytes (52 MB) copied, 0.100786 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0986518 s, 531 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112484 s, 466 MB/s 52428800 bytes (52 MB) copied, 0.0736551 s, 712 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826447 s, 634 MB/s 52428800 bytes (52 MB) copied, 0.116009 s, 452 MB/s , 0.0573812 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141975 s, 369 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0801341 s, 654 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144926 s, 362 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.220174 s, 238 MB/s , 0.157315 s, 333 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111187 s, 472 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0792005 s, 662 MB/s 52428800 bytes (52 MB) copied, 0.0539927 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404743 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290016 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0217524 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248141 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190454 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232538 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028542 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0174307 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279578 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379082 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241373 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277115 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307618 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347319 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258189 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437615 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0416188 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329244 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283003 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467856 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241121 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262306 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423357 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209827 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409394 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167375 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245844 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516002 s, 1.0 GB/s , 0.0532349 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558229 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527147 s, 995 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0858148 s, 611 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0913083 s, 574 MB/s 52428800 bytes (52 MB) copied, 0.112986 s, 464 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074173 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15428 s, 340 MB/s 52428800 bytes (52 MB) copied, 0.121123 s, 433 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392307 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0484779 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121955 s, 430 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107204 s, 489 MB/s 52428800 bytes (52 MB) copied, 0.166931 s, 314 MB/s 52428800 bytes (52 MB) copied, 0.134934 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132402 s, 396 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707846 s, 741 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0979651 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0928036 s, 565 MB/s , 0.140729 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104185 s, 503 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112307 s, 467 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594751 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551031 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361864 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554836 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276436 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389235 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214385 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266181 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178706 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430613 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510684 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395798 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200038 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293758 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486548 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299452 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385763 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341855 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217346 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247898 s, 2.1 GB/s , 0.0422335 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300032 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400831 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280084 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281713 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0439502 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198462 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349353 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033712 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345652 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267582 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190933 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468756 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208627 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350761 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317756 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599263 s, 875 MB/s 52428800 bytes (52 MB) copied, 0.0755854 s, 694 MB/s 52428800 bytes (52 MB) copied, 0.0777015 s, 675 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706904 s, 742 MB/s 52428800 bytes (52 MB) copied, 0.137607 s, 381 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748962 s, 700 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117047 s, 448 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135008 s, 388 MB/s 52428800 bytes (52 MB) copied, 0.135638 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170692 s, 307 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10312 s, 508 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150628 s, 348 MB/s 52428800 bytes (52 MB) copied, 0.109699 s, 478 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141376 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115394 s, 454 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127261 s, 412 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156633 s, 335 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103511 s, 507 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0786378 s, 667 MB/s 52428800 bytes (52 MB) copied, 0.156071 s, 336 MB/s 52428800 bytes (52 MB) copied, 0.0898812 s, 583 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157185 s, 334 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0883425 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126923 s, 413 MB/s 52428800 bytes (52 MB) copied, 0.0912094 s, 575 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0975514 s, 537 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101178 s, 518 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0993841 s, 528 MB/s 52428800 bytes (52 MB) copied, 0.0951252 s, 551 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689145 s, 761 MB/s 52428800 bytes (52 MB) copied, 0.0810516 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104506 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506445 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383676 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490283 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018114 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320168 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324061 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216218 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192182 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018298 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229412 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485977 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212113 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056684 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425096 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241201 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0760913 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0692567 s, 757 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236521 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166777 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047461 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551934 s, 950 MB/s 52428800 bytes (52 MB) copied, 0.0529719 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605023 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022776 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0754507 s, 695 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0766764 s, 684 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557659 s, 940 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0728266 s, 720 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826959 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668444 s, 784 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631238 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513937 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587641 s, 892 MB/s 52428800 bytes (52 MB) copied, 0.0572181 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339844 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335723 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247425 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198215 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418894 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193727 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244285 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425215 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190929 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252584 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037992 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241274 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290185 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01755 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028512 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251698 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331101 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203061 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156469 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159452 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154872 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195174 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0183021 s, 2.9 GB/s , 0.0256837 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255665 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371378 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262144 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036104 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0450897 s, 1.2 GB/s , 0.0388567 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496286 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431848 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467598 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656916 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417213 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205333 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0568808 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333674 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464677 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204134 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0728615 s, 720 MB/s 52428800 bytes (52 MB) copied, 0.0874055 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314269 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166439 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775358 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514961 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212647 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018095 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207318 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170606 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306417 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282851 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219081 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191294 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237076 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0206035 s, 2.5 GB/s , 0.0160413 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017798 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197996 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166833 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170731 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179044 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198131 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254145 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315129 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278061 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050732 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327762 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.06676 s, 785 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304722 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652925 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526309 s, 996 MB/s 52428800 bytes (52 MB) copied, 0.0515872 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313208 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612601 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221386 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391307 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548117 s, 957 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274649 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390279 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333606 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0268764 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551909 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02231 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521801 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303695 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293603 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215567 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269684 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165251 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335831 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01877 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153579 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178937 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169246 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228459 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01796 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183888 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412128 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052798 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365673 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397436 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0703097 s, 746 MB/s 52428800 bytes (52 MB) copied, 0.0650717 s, 806 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569518 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568216 s, 923 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402462 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629383 s, 833 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317873 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0742297 s, 706 MB/s 52428800 bytes (52 MB) copied, 0.0455465 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325372 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188688 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306444 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240394 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263137 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196477 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225297 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183725 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209073 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341692 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347028 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236136 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323818 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389351 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370026 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268875 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385282 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364375 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351841 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028654 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340267 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248199 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384413 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231071 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685779 s, 765 MB/s 52428800 bytes (52 MB) copied, 0.0550356 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036577 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177638 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332275 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022294 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311678 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184466 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194181 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037603 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184393 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502296 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188577 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402178 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191741 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0537195 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299973 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413349 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031784 s, 1.6 GB/s , 0.020827 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533192 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265747 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159376 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310314 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296302 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220135 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221037 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214414 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261843 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324477 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302255 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0451867 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394379 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398783 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680943 s, 770 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439101 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511589 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474269 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571078 s, 918 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476898 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101503 s, 517 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280012 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0246193 s, 2.1 GB/s , 0.0541311 s, 969 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732154 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211415 s, 2.5 GB/s , 0.0165758 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575751 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018902 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167855 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015596 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193959 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246934 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256627 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201431 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184118 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305659 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328312 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034412 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01602 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286346 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206763 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189718 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226779 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018229 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171823 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183483 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235856 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179946 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278985 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169309 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265631 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286959 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299808 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285599 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166117 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035804 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575978 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325739 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486229 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544586 s, 963 MB/s 52428800 bytes (52 MB) copied, 0.022958 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231898 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245557 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167367 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160323 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304396 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252873 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203534 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189429 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170171 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228791 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181442 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023345 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200559 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163067 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262787 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201665 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02968 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367245 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295395 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398563 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363998 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435205 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448304 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0442739 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519249 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354755 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707843 s, 741 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391737 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0762908 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064877 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0896762 s, 585 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236191 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0787024 s, 666 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219125 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0698701 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611179 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396389 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01676 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286502 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224563 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023522 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169619 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220133 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042649 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344876 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050378 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677456 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611084 s, 858 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791909 s, 662 MB/s 52428800 bytes (52 MB) copied, 0.0491496 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382816 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362749 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396094 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036776 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746239 s, 703 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374761 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483627 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0838029 s, 626 MB/s , 0.0727938 s, 720 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103036 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732035 s, 716 MB/s 52428800 bytes (52 MB) copied, 0.157996 s, 332 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147069 s, 356 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0964618 s, 544 MB/s 52428800 bytes (52 MB) copied, 0.210964 s, 249 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.088131 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137022 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128047 s, 409 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173897 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114807 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146119 s, 359 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130211 s, 403 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0895802 s, 585 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487546 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out , 0.0517678 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120204 s, 436 MB/s 52428800 bytes (52 MB) copied, 0.103813 s, 505 MB/s 52428800 bytes (52 MB) copied, 0.132652 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0944596 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139899 s, 375 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675893 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135547 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727884 s, 720 MB/s 52428800 bytes (52 MB) copied, 0.0537294 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371199 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309625 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331339 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170096 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224299 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203261 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195119 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021701 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204243 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299372 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.027492 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326405 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355887 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218445 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183988 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029293 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327794 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189659 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408457 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169471 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306176 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327165 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159213 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257628 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487712 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214279 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340101 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02563 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160293 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198839 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180347 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177974 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407422 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357078 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0354243 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034895 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328076 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388273 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414565 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550706 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473274 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364546 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338209 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241473 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392103 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278506 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205607 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256992 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241488 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166042 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205517 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194344 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223707 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252113 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297175 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397097 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225276 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03968 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279497 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183599 s, 2.9 GB/s , 0.0345073 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208377 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246602 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020497 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314846 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428173 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244148 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308476 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212081 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207074 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185696 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230166 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743156 s, 705 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02932 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047135 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0844419 s, 621 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763294 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0899689 s, 583 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036049 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304829 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602157 s, 871 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741648 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0955924 s, 548 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394566 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267479 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229618 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389232 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157578 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196969 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180267 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019701 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237071 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345513 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540756 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356624 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376777 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198342 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330427 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241688 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176199 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028082 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290525 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025984 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362099 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553894 s, 947 MB/s 52428800 bytes (52 MB) copied, 0.0427481 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204199 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350498 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405676 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0592939 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351044 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228224 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270601 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257865 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398871 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358136 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368829 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358447 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323417 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0324008 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309616 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283936 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269919 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337971 s, 1.6 GB/s , 0.0226814 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202449 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.020853 s, 2.5 GB/s , 0.0266005 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285053 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289595 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278401 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190964 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186469 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206281 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196819 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265004 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024883 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324778 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346101 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364783 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213589 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341568 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205257 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445634 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318836 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206783 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174389 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190708 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030288 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372357 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191375 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286961 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186375 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319361 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241098 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317031 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231485 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158669 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221201 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201649 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203998 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216537 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270396 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213374 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211209 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193862 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190823 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217125 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195355 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372152 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290832 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032545 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254341 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186187 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441102 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525935 s, 997 MB/s 52428800 bytes (52 MB) copied, 0.050793 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315982 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334995 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297001 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502084 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483021 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417307 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04103 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350349 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375475 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301256 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686331 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609347 s, 860 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558918 s, 938 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452186 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0651758 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048022 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498934 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412904 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357205 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323425 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271211 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445912 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436128 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294606 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269236 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170392 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284436 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188854 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176715 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275841 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271635 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04943 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323449 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236956 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477362 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245943 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221604 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039093 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388989 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240075 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226869 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257627 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388357 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222372 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250829 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0510043 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260947 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470538 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426299 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053075 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459125 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592198 s, 885 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041511 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502073 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462261 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0501703 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0839754 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0783437 s, 669 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679243 s, 772 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435261 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558265 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362131 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496656 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0895017 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551541 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246785 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343309 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281095 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184195 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266566 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032073 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306181 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357294 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567142 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758582 s, 691 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663622 s, 790 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199653 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0748776 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0965457 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118194 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677711 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127346 s, 412 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141271 s, 371 MB/s 52428800 bytes (52 MB) copied, 0.0672929 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107576 s, 487 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122193 s, 429 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124446 s, 421 MB/s 52428800 bytes (52 MB) copied, 0.127766 s, 410 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126308 s, 415 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143021 s, 367 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123753 s, 424 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131682 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142092 s, 369 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.101691 s, 516 MB/s 52428800 bytes (52 MB) copied, 0.153879 s, 341 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114124 s, 459 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111733 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101503 s, 517 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0995909 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102999 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703238 s, 746 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697976 s, 751 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0958399 s, 547 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0952271 s, 551 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0491447 s, 1.1 GB/s , 0.0569058 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225712 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242253 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11012 s, 476 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108345 s, 484 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256067 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345661 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197184 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349016 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221094 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218083 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241349 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223294 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217564 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217372 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242886 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0201667 s, 2.6 GB/s , 0.0321116 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332845 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250475 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172813 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333283 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019253 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400196 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428373 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405225 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469865 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538759 s, 973 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339121 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453619 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738356 s, 710 MB/s , 0.0632856 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700122 s, 749 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655378 s, 800 MB/s 52428800 bytes (52 MB) copied, 0.0770826 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0761072 s, 689 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372541 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742144 s, 706 MB/s 52428800 bytes (52 MB) copied, 0.101874 s, 515 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607382 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479853 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497086 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075548 s, 694 MB/s 52428800 bytes (52 MB) copied, 0.0669954 s, 783 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540426 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819795 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230518 s, 2.3 GB/s , 0.0674625 s, 777 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.071902 s, 729 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104003 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143036 s, 367 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419151 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0812213 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494339 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367156 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0371888 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507271 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411791 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366875 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229597 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249063 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03406 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357016 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292627 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274885 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277875 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474455 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244363 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226874 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239049 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214987 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028418 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299275 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222775 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389946 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296586 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352197 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0429817 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379424 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498439 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638403 s, 821 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429177 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564907 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509089 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286784 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598906 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499789 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240993 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415004 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315541 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749623 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233866 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0310856 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0392338 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249468 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426178 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367047 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285616 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378745 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0516681 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421201 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210919 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296237 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370303 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307119 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237858 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203771 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278268 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293961 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263495 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245394 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313189 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244189 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030153 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333679 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182478 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253474 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227441 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468529 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047683 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354887 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375272 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314506 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368862 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391843 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0184625 s, 2.8 GB/s , 0.0447562 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01778 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0382939 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469868 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370849 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203967 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588054 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390118 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234097 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020825 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176864 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348249 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311668 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276041 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243525 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345412 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022601 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443658 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446673 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437522 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046381 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0416459 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0636753 s, 823 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471034 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477817 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0694212 s, 755 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274243 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.049148 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604562 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452433 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449592 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405252 s, 1.3 GB/s , 0.0238585 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258426 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267011 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187207 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208319 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181504 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325401 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026113 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214984 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275934 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032042 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419005 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293214 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319977 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030448 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207851 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294986 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545899 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449289 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263423 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425487 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187246 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473794 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101751 s, 515 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116412 s, 450 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0888318 s, 590 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166244 s, 315 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134472 s, 390 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597359 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0897028 s, 584 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778819 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077939 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0808633 s, 648 MB/s 52428800 bytes (52 MB) copied, 0.108192 s, 485 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.056088 s, 935 MB/s , 0.0864192 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742887 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.071989 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549692 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048121 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687398 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0829809 s, 632 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250049 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181649 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196244 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325219 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398232 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190323 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255574 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256274 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328815 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316838 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237626 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036098 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217967 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0665074 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590439 s, 888 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380063 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.056236 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344655 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550205 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449462 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505608 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578425 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473346 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229364 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533848 s, 982 MB/s 52428800 bytes (52 MB) copied, 0.0459739 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437012 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038139 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174174 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214971 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169957 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016894 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182023 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225691 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268658 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181953 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191408 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195679 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222265 s, 2.4 GB/s , 0.027715 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297001 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198861 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266659 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389942 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411102 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331643 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0293997 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529487 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487269 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545094 s, 962 MB/s 52428800 bytes (52 MB) copied, 0.0466384 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480947 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694521 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0684062 s, 766 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373015 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0788698 s, 665 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0730342 s, 718 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0636714 s, 823 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703863 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469391 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421716 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344692 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497431 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340544 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331706 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298394 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019933 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238545 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173043 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179888 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022375 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294477 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273198 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230481 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370723 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025071 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270222 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217574 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222581 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025251 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203123 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323367 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384639 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353225 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244808 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225736 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318959 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218633 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375702 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414662 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375795 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0386089 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.079274 s, 661 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608239 s, 862 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615661 s, 852 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263936 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347827 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499895 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509146 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536715 s, 977 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346596 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488788 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374901 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317885 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393753 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023276 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220665 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379176 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051911 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256813 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262912 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274028 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268017 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042668 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248066 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364751 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295471 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263105 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185272 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220188 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219659 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206614 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026334 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258287 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282089 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535876 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324438 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227739 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258841 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046485 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256181 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0453865 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0318264 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265629 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0280362 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184039 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344315 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401136 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324886 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442267 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302313 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403556 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283631 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237929 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415602 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383988 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042657 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417949 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552985 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628617 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321962 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679968 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330455 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0949833 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107498 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595132 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222957 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174052 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544705 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01799 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194543 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275983 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211662 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433973 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035334 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03796 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344219 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359269 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539302 s, 972 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302787 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235745 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298954 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441431 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0598783 s, 876 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039755 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461713 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439741 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.071195 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584898 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0831025 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460792 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0683003 s, 768 MB/s , 0.104568 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0674914 s, 777 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635087 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0929069 s, 564 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0770708 s, 680 MB/s 52428800 bytes (52 MB) copied, 0.137727 s, 381 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642299 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0981629 s, 534 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0834209 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054074 s, 970 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668897 s, 784 MB/s 52428800 bytes (52 MB) copied, 0.0330641 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693156 s, 756 MB/s , 0.0684113 s, 766 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0361688 s, 1.4 GB/s , 0.0505129 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316525 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025418 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269539 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255032 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333963 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314738 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221983 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054783 s, 957 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321752 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498588 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306751 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606024 s, 865 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590746 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647189 s, 810 MB/s , 0.0519986 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0813756 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078203 s, 670 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771654 s, 679 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566522 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298503 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379964 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384101 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100785 s, 520 MB/s 52428800 bytes (52 MB) copied, 0.0762513 s, 688 MB/s 52428800 bytes (52 MB) copied, 0.0686369 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594024 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416403 s, 1.3 GB/s , 0.118084 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0877679 s, 597 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0961061 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.102265 s, 513 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105327 s, 498 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168516 s, 311 MB/s 52428800 bytes (52 MB) copied, 0.117197 s, 447 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102463 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0859724 s, 610 MB/s 52428800 bytes (52 MB) copied, 0.092928 s, 564 MB/s 52428800 bytes (52 MB) copied, 0.11653 s, 450 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0860508 s, 609 MB/s 52428800 bytes (52 MB) copied, 0.121495 s, 432 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668591 s, 784 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0838053 s, 626 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0925568 s, 566 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.084403 s, 621 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634919 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550089 s, 953 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381845 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0396287 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031274 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264552 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560058 s, 936 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479456 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504127 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302327 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379528 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387388 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429216 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601763 s, 871 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553693 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506187 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635153 s, 825 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604562 s, 867 MB/s 52428800 bytes (52 MB) copied, 0.0624548 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042579 s, 1.2 GB/s , 0.0412386 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0817939 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499654 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0425403 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508838 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337289 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259594 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678029 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336998 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232646 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278413 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218397 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269419 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273243 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287861 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234049 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0188981 s, 2.8 GB/s , 0.0226392 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227098 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019016 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281051 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255224 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345802 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206175 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258867 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278232 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04669 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327944 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387489 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304359 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285454 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253871 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197006 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198251 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231986 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206117 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488199 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0586977 s, 893 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647565 s, 810 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612202 s, 856 MB/s 52428800 bytes (52 MB) copied, 0.0593279 s, 884 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393434 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0903061 s, 581 MB/s 52428800 bytes (52 MB) copied, 0.0659519 s, 795 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034106 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746802 s, 702 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654927 s, 801 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065681 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718901 s, 729 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524319 s, 1000 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360512 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288712 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562407 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429164 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0880018 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426024 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274591 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506351 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429167 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042202 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0261481 s, 2.0 GB/s , 0.0431684 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456776 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381918 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018629 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196319 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172883 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397411 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233962 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406622 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440972 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245486 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042137 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233567 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268162 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254673 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738767 s, 710 MB/s 52428800 bytes (52 MB) copied, 0.0863263 s, 607 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363896 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0244053 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361161 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410313 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254391 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409098 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203295 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339477 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0421041 s, 1.2 GB/s , 0.0211877 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468002 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349567 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609199 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311262 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213394 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247327 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258915 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207936 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184646 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192955 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226309 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020237 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250851 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270604 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406419 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336978 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250059 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252635 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241422 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332311 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388811 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258254 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229235 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212152 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270149 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169496 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234804 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255973 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019994 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271142 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028571 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273755 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265128 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249301 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214624 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187469 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041548 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042905 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268186 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644171 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458497 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479944 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281722 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186938 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290548 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181356 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305756 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368546 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246906 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176172 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247713 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181912 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.020532 s, 2.6 GB/s , 0.0175771 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320603 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278745 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206858 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022037 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036333 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366466 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287461 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304681 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332022 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285426 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327462 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310878 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530029 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153179 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359172 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413131 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020366 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334222 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235714 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388749 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0591241 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379739 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430754 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220234 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470731 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320117 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023029 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567327 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187037 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233903 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181082 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231211 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348313 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224247 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196213 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200677 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028032 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197941 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025042 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304302 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177305 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401278 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207491 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322805 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.033159 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220026 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215648 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315123 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558019 s, 940 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176423 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274392 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196132 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267105 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381275 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380651 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219019 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600027 s, 874 MB/s 52428800 bytes (52 MB) copied, 0.0629534 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048929 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506936 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363409 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663906 s, 790 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441194 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355682 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0796473 s, 658 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343043 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574915 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043242 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0768818 s, 682 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432977 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296418 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539655 s, 972 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335832 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545851 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219498 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268517 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0636442 s, 824 MB/s 52428800 bytes (52 MB) copied, 0.0294827 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0846636 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0692592 s, 757 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0745751 s, 703 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575075 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550334 s, 953 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776841 s, 675 MB/s 52428800 bytes (52 MB) copied, 0.0670328 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810895 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591602 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out , 0.0756481 s, 693 MB/s 52428800 bytes (52 MB) copied, 0.0589439 s, 889 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731847 s, 716 MB/s 52428800 bytes (52 MB) copied, 0.0807688 s, 649 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677741 s, 774 MB/s 52428800 bytes (52 MB) copied, 0.0656329 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521391 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741225 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546745 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510138 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502015 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462488 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346933 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351063 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289536 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241851 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277785 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030209 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361186 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342059 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383928 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470261 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364252 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511801 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356838 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330508 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036366 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283618 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429302 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337775 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584725 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458446 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395394 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363304 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357717 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078041 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529082 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540074 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491647 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276592 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569816 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.083525 s, 628 MB/s 52428800 bytes (52 MB) copied, 0.0809412 s, 648 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0719887 s, 728 MB/s 52428800 bytes (52 MB) copied, 0.0805376 s, 651 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595577 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534914 s, 980 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387889 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789584 s, 664 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381559 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0906943 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386545 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127237 s, 412 MB/s 52428800 bytes (52 MB) copied, 0.0608652 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775705 s, 676 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0638388 s, 821 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152957 s, 343 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0898996 s, 583 MB/s , 0.135357 s, 387 MB/s 52428800 bytes (52 MB) copied, 0.0705922 s, 743 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0902131 s, 581 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062436 s, 840 MB/s 52428800 bytes (52 MB) copied, 0.103092 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0974746 s, 538 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298069 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0738637 s, 710 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725769 s, 722 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385179 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10566 s, 496 MB/s 52428800 bytes (52 MB) copied, 0.12207 s, 429 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483748 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481403 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444523 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049405 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263723 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167799 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558529 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390358 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0407549 s, 1.3 GB/s , 0.020197 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037509 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431152 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264863 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572537 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367684 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062413 s, 840 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278546 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0253941 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464412 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687146 s, 763 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319699 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0481127 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217524 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0202264 s, 2.6 GB/s , 0.0291035 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176269 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310489 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240119 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229442 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500461 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330288 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0256864 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449624 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210348 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241627 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320856 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610511 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353175 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289641 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322457 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342721 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294604 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276862 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446572 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417917 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0482195 s, 1.1 GB/s , 0.0310314 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362018 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498932 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0444916 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325003 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383043 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332773 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343692 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365793 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478107 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342425 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390143 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268002 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325015 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0565405 s, 927 MB/s , 0.0476112 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548389 s, 956 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476012 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227827 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245265 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200529 s, 2.6 GB/s /mnt/lustre/f45.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x8 lmm_fid: [0x2000059f1:0x8:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x1a5:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' llite.lustre-ffff8800aadd2000.stats= snapshot_time 1746071766.923001982 secs.nsecs start_time 1746071706.420084949 secs.nsecs elapsed_time 60.502917033 secs.nsecs read_bytes 1150 samples [bytes] 1048576 1048576 1205862400 1264438371942400 read 140700 samples [usecs] 151 248284 149979240 2921927389708 open 2814 samples [usecs] 3 40473 221126 3748548020 close 2814 samples [usecs] 5 51178 428998 6004202126 seek 2813 samples [usecs] 2 2998 20417 14939907 inode_permission 5626 samples [usecs] 1 17080 437473 1092807679 opencount 2814 samples [reqs] 1 2798 3915817 7305579415 openclosetime 2797 samples [usecs] 9 199804 56407320 2442899234024 pcc_attach 1 samples [reqs] pcc_hit_bytes 2791 samples [bytes] 52428800 52428800 146328780800 7671842382807040000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 llite.lustre-ffff8800aae60000.stats= snapshot_time 1746071766.923111243 secs.nsecs start_time 1746071706.420101794 secs.nsecs elapsed_time 60.503009449 secs.nsecs read_bytes 1804 samples [bytes] 114688 1048576 1887436800 1977613664911360 read 143250 samples [usecs] 141 402380 146392837 2684436362683 ioctl 1 samples [reqs] open 2866 samples [usecs] 4 61411 745422 30678212080 close 2866 samples [usecs] 4 25715 434988 4369915332 seek 2864 samples [usecs] 2 2665 18467 8518009 setattr 1 samples [usecs] 4914 4914 4914 24147396 getattr 32 samples [usecs] 110 7361 19574 101587688 mknod 1 samples [usecs] 11578 11578 11578 134050084 inode_permission 11442 samples [usecs] 1 13487 1131371 1692955723 opencount 2867 samples [reqs] 1 2866 4108412 7851173422 openclosetime 2865 samples [usecs] 13 350482 56201727 2327668453777 pcc_attach 1 samples [reqs] pcc_hit_bytes 2829 samples [bytes] 52428800 52428800 148321075200 7776295987445760000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 attach 2 detach 0 auto_attach 0 llite.lustre-ffff8800aadd2000.pcc_async_threshold=1073741824 llite.lustre-ffff8800aae60000.pcc_async_threshold=1073741824 /mnt/pcc.d45.sanity-pcc.2 is idle now /mnt/pcc.d45.sanity-pcc is idle now PASS 45 (67s) == sanity-pcc test 46: Verify PCC mode setting works correctly ========================================================== 23:56:10 (1746071770) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0982836 s, 640 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d46.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: f5eac3f2-1a04-4ba2-afa0-2091e082b4cf Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f46.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f5eac3f2-1a04-4ba2-afa0-2091e082b4cf (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d46.sanity-pcc/d46.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} running as uid/gid/euid/egid 500/500/500/500, groups: 0 [id] uid=500(sanityusr) gid=500(sanityusr) groups=500(sanityusr),0(root) Mode: 0400 llite.lustre-ffff8800aae60000.pcc_mode=0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f46.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ llite.lustre-ffff8800aae60000.pcc_mode=0400 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f46.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ llite.lustre-ffff8800aae60000.pcc_mode=0004 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 0 [stat] [/mnt/lustre/f46.sanity-pcc] File: '/mnt/lustre/f46.sanity-pcc' Size: 6 Blocks: 1 IO Block: 4194304 regular file Device: 2c54f966h/743766374d Inode: 144115574371254282 Links: 1 Access: (0664/-rw-rw-r--) Uid: ( 0/ root) Gid: ( 0/ root) Access: 2025-04-30 23:56:12.000000000 -0400 Modify: 2025-04-30 23:56:11.000000000 -0400 Change: 2025-04-30 23:56:11.000000000 -0400 Birth: - running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] llite.lustre-ffff8800aae60000.pcc_mode=0040 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] File: '/mnt/lustre/f46.sanity-pcc' Size: 6 Blocks: 4 IO Block: 4194304 regular file Device: 2c54f966h/743766374d Inode: 144115574371254282 Links: 1 Access: (0660/-rw-rw----) Uid: ( 0/ root) Gid: ( 0/ root) Access: 2025-04-30 23:56:12.000000000 -0400 Modify: 2025-04-30 23:56:11.000000000 -0400 Change: 2025-04-30 23:56:12.000000000 -0400 Birth: - file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] llite.lustre-ffff8800aae60000.pcc_mode=0400 llite.lustre-ffff8800aae60000.pcc_mode=0400 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d46.sanity-pcc is idle now PASS 46 (5s) == sanity-pcc test 47: mtime should be kept once file attached into PCC ========================================================== 23:56:15 (1746071775) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.109814 s, 573 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d47.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: bf2920c7-3f32-433c-932d-f044b77c26f9 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f47.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=bf2920c7-3f32-433c-932d-f044b77c26f9 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d47.sanity-pcc/d47.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} QQQQQ /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/f47.sanity-pcc | grep -E -c 'type: readonly' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d47.sanity-pcc is idle now PASS 47 (7s) == sanity-pcc test 48: PCC state should check whether the file in local PCC cache ========================================================== 23:56:22 (1746071782) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.116378 s, 541 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d48.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 500f7215-90d6-4f4a-a5d0-1f70c3866b7f Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f48.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=500f7215-90d6-4f4a-a5d0-1f70c3866b7f (extents) (64bit) (large files) (huge files) QQQQQ file: /mnt/lustre/f48.sanity-pcc, type: readonly, PCC_file: /d48.sanity-pcc/59fd/0x2000059f1:0xc:0x0 (unlinked), open_count: 0, flags: 4 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d48.sanity-pcc is idle now PASS 48 (5s) == sanity-pcc test 96: Auto attach from multiple read process on a node ========================================================== 23:56:27 (1746071787) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.12453 s, 505 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d96.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: c403c498-e8f8-4251-89b3-4e3a79ba3bac Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f96.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=c403c498-e8f8-4251-89b3-4e3a79ba3bac (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d96.sanity-pcc/d96.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800aadd2000.pcc_async_threshold=1G llite.lustre-ffff8800aae60000.pcc_async_threshold=1G 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.480692 s, 109 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.527114 s, 99.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.534087 s, 98.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043694 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.658586 s, 79.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.639241 s, 82.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.683952 s, 76.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205835 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023184 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598965 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216762 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347808 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300717 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034257 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414049 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239264 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336338 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451176 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434651 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035279 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577053 s, 909 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247813 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373781 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267068 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216676 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018288 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293958 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296762 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197418 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297791 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450982 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227267 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266011 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210491 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231929 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276396 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282455 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192224 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294123 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268442 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02192 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236047 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017469 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171163 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172911 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199815 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022364 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220037 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196553 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196723 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303569 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200406 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447417 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459359 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179402 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031648 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233773 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233138 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363103 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339256 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197022 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262586 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323697 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024814 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214575 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241189 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216943 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021615 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221365 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252167 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323813 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293088 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281715 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250758 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209862 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292177 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249084 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246843 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233841 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284265 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178464 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414717 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188584 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540213 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403175 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443059 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525637 s, 997 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298494 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366262 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044266 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400626 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289543 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287477 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196206 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255176 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230531 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210234 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211235 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024137 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240309 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158353 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283949 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170684 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317198 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273374 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225795 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246006 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213593 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282492 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436222 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260809 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380944 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293754 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209225 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428019 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352087 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056955 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391969 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467638 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360617 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222237 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255982 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328481 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021326 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288695 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025623 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021579 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380249 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256987 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358633 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449872 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168226 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408932 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283383 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250537 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205582 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024412 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227252 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184618 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361139 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253493 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264396 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216284 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167242 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020588 s, 2.5 GB/s , 0.0221003 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017781 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249666 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187224 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016594 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021303 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01669 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204503 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201589 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195755 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200774 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01945 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019249 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255353 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404323 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248367 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166127 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331384 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259474 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315536 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019453 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212713 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163336 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254384 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180977 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016228 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301953 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173789 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243122 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031288 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261081 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464384 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398983 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230008 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204269 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223577 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208078 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030332 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258473 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307445 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260394 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168857 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207604 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01836 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273742 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043846 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496516 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359858 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031931 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337444 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428546 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256507 s, 2.0 GB/s , 0.0330838 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244116 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184937 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427921 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335001 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235798 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022279 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266366 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244607 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425364 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282258 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501915 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271155 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017964 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392038 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035773 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309205 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312179 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253524 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284778 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514326 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345033 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313252 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397368 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395996 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239785 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239697 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582912 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238579 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700765 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472079 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303685 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387675 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362875 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328219 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464805 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236099 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362277 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196086 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285567 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198435 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451316 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385723 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255485 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638462 s, 821 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400442 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030288 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265725 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209527 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286728 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230833 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381361 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352202 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435295 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278301 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331675 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545753 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233109 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189916 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193759 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210764 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235499 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347528 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344486 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518799 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491435 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261058 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472403 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355574 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418867 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350229 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227938 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212635 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265006 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357936 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193388 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224289 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552571 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370555 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273835 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355129 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266388 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278053 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301919 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435584 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476994 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022609 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221554 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530706 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263216 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209964 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231321 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206708 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265926 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206572 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233817 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306366 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282702 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489414 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390253 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167452 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623195 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577422 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457725 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588992 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020542 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281712 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188932 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019987 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192895 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256599 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329494 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349518 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376728 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248013 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273389 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306566 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415165 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260543 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291531 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304564 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382501 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023261 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368563 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289909 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361118 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025476 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389315 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032224 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567707 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618649 s, 847 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0330997 s, 1.6 GB/s , 0.0335573 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430832 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234131 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540786 s, 969 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207196 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03482 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471628 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383691 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293099 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349534 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351588 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174751 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214572 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256239 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058739 s, 893 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377659 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358792 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414582 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343379 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467619 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223905 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208284 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424177 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419721 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433277 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291659 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436571 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496696 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530273 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208581 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225276 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172353 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328089 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439325 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372433 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246707 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360937 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216053 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222935 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285457 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157259 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276928 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267281 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291916 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362891 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271935 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298757 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203861 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026175 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221439 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240719 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239254 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274446 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024655 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293489 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254702 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188509 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328525 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262919 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189338 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198957 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357073 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239777 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223044 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241563 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314936 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353085 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289938 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024727 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275908 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334964 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201149 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421634 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347516 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468474 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270753 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200461 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254349 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281988 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02929 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229734 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186307 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320123 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374581 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362927 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369397 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209751 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349112 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287961 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303121 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203247 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187838 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207357 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174952 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184132 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173656 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303411 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307767 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305748 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362643 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294836 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253473 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347744 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453599 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278186 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380599 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226786 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334498 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240279 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302644 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258366 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253598 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275663 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160498 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364182 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294968 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389172 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201234 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553753 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433659 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247114 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288928 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369156 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214133 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374773 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024311 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039167 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430072 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302701 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212643 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214785 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286388 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042207 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277949 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211596 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024977 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249983 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492847 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048863 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509904 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220709 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176949 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290185 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419676 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319789 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185834 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482477 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575557 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393583 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408568 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312592 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263326 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313333 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325933 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284929 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216576 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295035 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395577 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210828 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256641 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263984 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158353 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235626 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268788 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225286 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217899 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162714 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225685 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230256 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407393 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294634 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218418 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206277 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307598 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176443 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234187 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197451 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367568 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277172 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223903 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0403924 s, 1.3 GB/s , 0.0459024 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264547 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273216 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226414 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019837 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207519 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280941 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294235 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179866 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234883 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023301 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251237 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294728 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320693 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329716 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024573 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282048 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021697 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018642 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186845 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190696 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274305 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260648 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023566 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318583 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256675 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301715 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205387 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295573 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179767 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022167 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198494 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249384 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306169 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326284 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324315 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218619 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183472 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02227 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322526 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356712 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174917 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197519 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227612 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245108 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240826 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233972 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258173 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02444 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179068 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235686 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319552 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0301631 s, 1.7 GB/s , 0.0295318 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032334 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181765 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203346 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338057 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284737 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212853 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293522 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186246 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284588 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049769 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029741 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389571 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282617 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188933 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319336 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281149 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206388 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199289 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041189 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284178 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280912 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021969 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352777 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242905 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243546 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019528 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038031 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267926 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210183 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400687 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026017 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356977 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024985 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260019 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198149 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227462 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404053 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469614 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364714 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289267 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338384 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286136 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237417 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021135 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320508 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316174 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549682 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422062 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311562 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272526 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204862 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233616 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558785 s, 938 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334987 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316961 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428647 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316555 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277714 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020448 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540722 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382024 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372679 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567675 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403917 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542323 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0712585 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269556 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358229 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429724 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283681 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044497 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367924 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367913 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686335 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547489 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277128 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423195 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244304 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219272 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350213 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027448 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172926 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556028 s, 943 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422851 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509284 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394761 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467176 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252222 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372462 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237741 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213084 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289013 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302101 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380161 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510626 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354041 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298755 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228938 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331374 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276945 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019905 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387783 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252712 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326953 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280669 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231437 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220414 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213737 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240812 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423821 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360222 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306708 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281981 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260837 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561789 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263822 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274098 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289037 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241835 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191852 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393571 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028662 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022711 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313482 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402498 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411604 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287097 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286727 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0356135 s, 1.5 GB/s , 0.0301905 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299618 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422568 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347939 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035614 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342498 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424309 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225287 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585087 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342445 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246542 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374893 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196191 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260775 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031345 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323586 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420205 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311232 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226263 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024761 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325547 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262013 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218396 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275793 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354562 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043346 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219324 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305024 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309222 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335756 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576712 s, 909 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405181 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414407 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022641 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272644 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202688 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241318 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365341 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320567 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652645 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291612 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029364 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284398 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0645952 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285525 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024753 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020143 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201554 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296695 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377012 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483856 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344696 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352643 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019611 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189797 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178177 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284501 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270022 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238381 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216285 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228025 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248277 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238886 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290191 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348972 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226384 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228769 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176669 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334548 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284141 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193048 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352488 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328588 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223271 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243546 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202621 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180003 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021813 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191127 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203382 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238522 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244747 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024856 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204318 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208765 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173779 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176856 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176847 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237887 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208211 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198039 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215289 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274417 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208686 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228878 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243627 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210975 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216184 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266752 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211257 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165091 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229891 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219793 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268144 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319662 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318761 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201029 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171532 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371604 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303341 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330326 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194014 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244807 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226171 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232256 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246849 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303341 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337881 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232186 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313683 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020376 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254551 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022374 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232088 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281936 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212749 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242519 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388944 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190379 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0209147 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279301 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249219 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214812 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259505 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178236 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218169 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301645 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289687 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360698 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242354 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246935 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280326 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208917 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235885 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184232 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020654 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280712 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361272 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238352 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253734 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020736 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188864 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278182 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181886 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290293 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183666 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212002 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170609 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260219 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408591 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246889 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030905 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230818 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185293 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173252 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224412 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225418 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292814 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206732 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210142 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246178 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249489 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220197 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021073 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203503 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204938 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238238 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244441 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273479 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271361 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222312 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230974 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283581 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020403 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202353 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293321 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230912 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381331 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269745 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335339 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250352 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162615 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291996 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264402 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214573 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192828 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244106 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291627 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293225 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279812 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371296 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187052 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214337 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296976 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213871 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168208 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325342 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271248 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195572 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254138 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231526 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200757 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159452 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192122 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017108 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261137 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270453 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217745 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241744 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255596 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234693 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255312 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176943 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232262 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0144609 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196641 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178124 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211511 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277406 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239264 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180436 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188069 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024652 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177534 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019192 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197566 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205929 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300783 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245399 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186591 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285389 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024146 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244185 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200754 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171474 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298522 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231735 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186575 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338945 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301555 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170434 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254768 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392019 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186401 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027662 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227085 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221929 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249118 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183327 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166362 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372116 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191098 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381192 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220652 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022785 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216093 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288355 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206953 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250077 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257734 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281307 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322059 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354401 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296163 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214094 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040066 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201405 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189426 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181132 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287833 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315447 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286697 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342398 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363551 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603562 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416486 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021026 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225435 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325028 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445789 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191104 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282347 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323007 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264485 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319395 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377232 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247417 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031899 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369844 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214326 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234275 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195352 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446177 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182838 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544373 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336367 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322097 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360826 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330682 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288308 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218886 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236248 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369057 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213199 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326468 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241845 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281761 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411631 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375271 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391554 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192778 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310719 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244696 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326606 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311737 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025835 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024529 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315678 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242683 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372569 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490932 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309287 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210343 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537583 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597741 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226674 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041296 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054018 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201571 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040326 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277655 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305311 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285926 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300934 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274083 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385138 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221892 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228051 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431087 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236416 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390527 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307908 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026773 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322643 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547372 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244867 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212042 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218128 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262111 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03177 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229564 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203356 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738042 s, 710 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217801 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173693 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266919 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442091 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226993 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305487 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333464 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030804 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381962 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311714 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414615 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289482 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387695 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315492 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348516 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020395 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278285 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0374598 s, 1.4 GB/s , 0.0335554 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263995 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325195 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380491 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398488 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516937 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362911 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04102 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029311 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307528 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215041 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360305 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565655 s, 927 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038791 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411563 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381144 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344237 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293802 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265583 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020939 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189188 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026824 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230271 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265709 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385112 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302094 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303322 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351888 s, 1.5 GB/s Finish ======== detach: must specify one or more FIDs Detach given files from the Persistent Client Cache. usage: lfs pcc detach [-k] FILE [...] lfs commands used to interact with PCC features: lfs pcc attach - attach given files to Persistent Client Cache lfs pcc attach_fid - attach given files into PCC by FID(s) lfs pcc state - display the PCC state for given files lfs pcc detach - detach given files from Persistent Client Cache lfs pcc detach_fid - detach given files from PCC by FID(s) lfs pcc delete - delete the PCC layout componenet for given files lfs pcc pin - pin give files for PCC lfs pcc unpin - unpin given files for PCC llite.lustre-ffff8800aadd2000.pcc_async_threshold=1073741824 llite.lustre-ffff8800aae60000.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d96.sanity-pcc is idle now PASS 96 (65s) == sanity-pcc test 97: two mmap I/O and layout lock cancel ========================================================== 23:57:32 (1746071852) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.121997 s, 516 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d97.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 16854ae6-ca56-4bb1-8d5a-5528d2947293 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f97.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=16854ae6-ca56-4bb1-8d5a-5528d2947293 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d97.sanity-pcc/d97.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800aadd2000.pcc_async_threshold=1G llite.lustre-ffff8800aae60000.pcc_async_threshold=1G 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.567145 s, 92.4 MB/s T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... llite.lustre-ffff8800aadd2000.pcc_async_threshold=1073741824 llite.lustre-ffff8800aae60000.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d97.sanity-pcc is idle now PASS 97 (125s) == sanity-pcc test 98: racer between auto attach and mmap I/O ========================================================== 23:59:37 (1746071977) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.100605 s, 625 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d98.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: cb4f14a8-ef20-402d-b5c6-309831210482 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f98.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=cb4f14a8-ef20-402d-b5c6-309831210482 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d98.sanity-pcc/d98.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800aadd2000.pcc_async_threshold=0 llite.lustre-ffff8800aae60000.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.436832 s, 120 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.319283 s, 164 MB/s 52428800 bytes (52 MB) copied, 0.397803 s, 132 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.397528 s, 132 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298347 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0191267 s, 2.7 GB/s , 0.019466 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398491 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.454157 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.30989 s, 169 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.516827 s, 101 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.424936 s, 123 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.538533 s, 97.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.454958 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.092005 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.311636 s, 168 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256695 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127322 s, 412 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.446603 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.446622 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.227068 s, 231 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.539166 s, 97.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.677488 s, 77.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.248323 s, 211 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.226284 s, 232 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375241 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311028 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321417 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356159 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221399 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450593 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046211 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403267 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355349 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253944 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206186 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220019 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309899 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476637 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672284 s, 780 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550047 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358156 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286519 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193431 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380196 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317823 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268443 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209969 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305462 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030417 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341337 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232661 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299771 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264707 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353915 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261285 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368752 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349257 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205384 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214684 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.464347 s, 113 MB/s , 0.464404 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.357537 s, 147 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.279675 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.389752 s, 135 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0786048 s, 667 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.223982 s, 234 MB/s 52428800 bytes (52 MB) copied, 0.0737247 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276233 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.690931 s, 75.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288465 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172685 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268585 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322756 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200459 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293835 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167909 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023322 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333199 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263988 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244969 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02245 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363922 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339029 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252239 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288395 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523456 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397421 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0772275 s, 679 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297514 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457017 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310302 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452891 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046121 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278075 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.206517 s, 254 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.382528 s, 137 MB/s , 0.710334 s, 73.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.579574 s, 90.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.460427 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.229094 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.558448 s, 93.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.362835 s, 144 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0844517 s, 621 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.907428 s, 57.8 MB/s , 0.797913 s, 65.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369022 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606732 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479308 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326173 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554823 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588279 s, 891 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258617 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481808 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413633 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313758 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315706 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404131 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262478 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314857 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357321 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248779 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329927 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335157 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247874 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375788 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434739 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347967 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.657544 s, 79.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.709511 s, 73.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.611253 s, 85.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138329 s, 379 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.611504 s, 85.7 MB/s 52428800 bytes (52 MB) copied, 0.325635 s, 161 MB/s 52428800 bytes (52 MB) copied, 0.506824 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.391398 s, 134 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.343994 s, 152 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.546827 s, 95.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.402002 s, 130 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.291462 s, 180 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.318178 s, 165 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.743682 s, 70.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.378756 s, 138 MB/s , 0.746069 s, 70.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.477681 s, 110 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.478904 s, 109 MB/s 52428800 bytes (52 MB) copied, 0.399476 s, 131 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.41545 s, 126 MB/s 52428800 bytes (52 MB) copied, 0.310734 s, 169 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.289882 s, 181 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.428909 s, 122 MB/s 52428800 bytes (52 MB) copied, 0.27886 s, 188 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211173 s, 248 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.438127 s, 120 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.344506 s, 152 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160348 s, 327 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.574304 s, 91.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.578539 s, 90.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.684114 s, 76.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.32512 s, 161 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118382 s, 443 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.533417 s, 98.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.406947 s, 129 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.415909 s, 126 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470499 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.299633 s, 175 MB/s , 0.148609 s, 353 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.328665 s, 160 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214185 s, 245 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.440758 s, 119 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.445129 s, 118 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126588 s, 414 MB/s , 0.125604 s, 417 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248331 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.422818 s, 124 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.332886 s, 157 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.33344 s, 157 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0971488 s, 540 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.360684 s, 145 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0980297 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.298311 s, 176 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0949606 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153492 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364032 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.363005 s, 144 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.311909 s, 168 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.365379 s, 143 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.243766 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.343564 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.349205 s, 150 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.360169 s, 146 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.273159 s, 192 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606708 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.256566 s, 204 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.257118 s, 204 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.260319 s, 201 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.160932 s, 326 MB/s 52428800 bytes (52 MB) copied, 0.101067 s, 519 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.269994 s, 194 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.302551 s, 173 MB/s , 0.300529 s, 174 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.304356 s, 172 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461789 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.290028 s, 181 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.30226 s, 173 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.264711 s, 198 MB/s 52428800 bytes (52 MB) copied, 0.265735 s, 197 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438345 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498848 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136939 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.445465 s, 118 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.447488 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.354107 s, 148 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668004 s, 785 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163511 s, 321 MB/s 52428800 bytes (52 MB) copied, 0.395518 s, 133 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.403949 s, 130 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.28743 s, 182 MB/s , 0.115131 s, 455 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116061 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.227317 s, 231 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.378225 s, 139 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.623958 s, 84.0 MB/s 52428800 bytes (52 MB) copied, 0.388659 s, 135 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.871995 s, 60.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.402528 s, 130 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.669849 s, 78.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.136978 s, 383 MB/s 52428800 bytes (52 MB) copied, 0.343708 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.250206 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0875025 s, 599 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0909439 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.379323 s, 138 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211796 s, 248 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.4334 s, 121 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.308448 s, 170 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.339295 s, 155 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338962 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375209 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.263071 s, 199 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.309917 s, 169 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.313245 s, 167 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.293434 s, 179 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.329802 s, 159 MB/s , 0.331556 s, 158 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.079176 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407221 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.501224 s, 105 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233907 s, 224 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.430479 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.312548 s, 168 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.397478 s, 132 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.401064 s, 131 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0882615 s, 594 MB/s , 0.0373364 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.519131 s, 101 MB/s , 0.405499 s, 129 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.40429 s, 130 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0951064 s, 551 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.267394 s, 196 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.431259 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.300383 s, 175 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281829 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032236 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399149 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135477 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332094 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182979 s, 287 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.360344 s, 145 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.279436 s, 188 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.210799 s, 249 MB/s , 0.210788 s, 249 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131752 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.341152 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.342084 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.34427 s, 152 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12819 s, 409 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.399501 s, 131 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.544592 s, 96.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.465126 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144525 s, 363 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.36339 s, 144 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130554 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.552464 s, 94.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080646 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0754389 s, 695 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252388 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.234465 s, 224 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18038 s, 291 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.456415 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15843 s, 331 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.42858 s, 122 MB/s , 0.327641 s, 160 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.432382 s, 121 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680674 s, 770 MB/s , 0.107281 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499166 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.392133 s, 134 MB/s , 0.215194 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139192 s, 377 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.272604 s, 192 MB/s , 0.272655 s, 192 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196137 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741572 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0766838 s, 684 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.407317 s, 129 MB/s 52428800 bytes (52 MB) copied, 0.0825677 s, 635 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.287711 s, 182 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0953207 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.288125 s, 182 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.289372 s, 181 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14714 s, 356 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148298 s, 354 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.457476 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.185027 s, 283 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.291631 s, 180 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.392339 s, 134 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.482518 s, 109 MB/s 52428800 bytes (52 MB) copied, 0.382706 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.279263 s, 188 MB/s , 0.166986 s, 314 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.281572 s, 186 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.275753 s, 190 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.277538 s, 189 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.339066 s, 155 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.207006 s, 253 MB/s 52428800 bytes (52 MB) copied, 0.12367 s, 424 MB/s , 0.208482 s, 251 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175537 s, 299 MB/s 52428800 bytes (52 MB) copied, 0.282427 s, 186 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171263 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.30137 s, 174 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.298937 s, 175 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.247117 s, 212 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.386565 s, 136 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.387653 s, 135 MB/s 52428800 bytes (52 MB) copied, 0.252275 s, 208 MB/s llite.lustre-ffff8800aadd2000.pcc_async_threshold=1073741824 llite.lustre-ffff8800aae60000.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d98.sanity-pcc is idle now PASS 98 (65s) == sanity-pcc test 99: race among unlink | mmap read | write | detach for PCC-RO file ========================================================== 00:00:42 (1746072042) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.109164 s, 576 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d99.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 7b65fa16-731c-4280-ba9e-2be203d98c8d Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f99.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=7b65fa16-731c-4280-ba9e-2be203d98c8d (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.543063 s, 96.5 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.970592 s, 54.0 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.09772 s, 47.8 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.608212 s, 86.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.816277 s, 64.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.5117 s, 102 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.619483 s, 84.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240059 s, 2.2 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.577958 s, 90.7 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.01909 s, 51.4 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.359545 s, 146 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.566685 s, 92.5 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.329229 s, 159 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.322351 s, 163 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.852877 s, 61.5 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.46474 s, 113 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275209 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.45762 s, 115 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.895807 s, 58.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380894 s, 1.4 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.750676 s, 69.8 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.982412 s, 53.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.376556 s, 139 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.598161 s, 87.7 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.64156 s, 81.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.709066 s, 73.9 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.718574 s, 73.0 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.634544 s, 82.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.717164 s, 73.1 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.681205 s, 77.0 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444632 s, 1.2 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.43149 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.695555 s, 75.4 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.770513 s, 68.0 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259979 s, 2.0 GB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.645422 s, 81.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.0282 s, 51.0 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.642546 s, 81.6 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035982 s, 1.5 GB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.385796 s, 136 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.910681 s, 57.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019892 s, 2.6 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379095 s, 1.4 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.854607 s, 61.3 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.649405 s, 80.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.375646 s, 140 MB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.515833 s, 102 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.575689 s, 91.1 MB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.455951 s, 115 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.487695 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299686 s, 1.7 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.510529 s, 103 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351565 s, 1.5 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.311887 s, 168 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.710364 s, 73.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193825 s, 2.7 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.511632 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219214 s, 2.4 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.685119 s, 76.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306343 s, 1.7 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.476842 s, 110 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.571661 s, 91.7 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.71278 s, 73.6 MB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.490994 s, 107 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.611187 s, 85.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.610069 s, 85.9 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.499791 s, 105 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022046 s, 2.4 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.376872 s, 139 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.4855 s, 108 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.695105 s, 75.4 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.390816 s, 134 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.653646 s, 80.2 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.404741 s, 130 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.976633 s, 53.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.3338 s, 157 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.60099 s, 87.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020392 s, 2.6 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.502156 s, 104 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.387701 s, 135 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359264 s, 1.5 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.469832 s, 112 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.979872 s, 53.5 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.368916 s, 142 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.523339 s, 100 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.718072 s, 73.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.789993 s, 66.4 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.871482 s, 60.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312554 s, 1.7 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.590832 s, 88.7 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.774075 s, 67.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.65137 s, 80.5 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.371349 s, 141 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.956011 s, 54.8 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.299378 s, 175 MB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.726535 s, 72.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.855877 s, 61.3 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.664052 s, 79.0 MB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.932856 s, 56.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242103 s, 2.2 GB/s Read /mnt/lustre/f99.sanity-pcc ... ==== DONE ==== Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.633305 s, 82.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.895488 s, 58.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.904363 s, 58.0 MB/s ==== DONE WIAT ==== osc.lustre-OST0000-osc-ffff8800aadd2000.rpc_stats= snapshot_time: 1746072107.450485036 secs.nsecs start_time: 1746071566.333139792 secs.nsecs elapsed_time: 541.117345244 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 4 8 8 | 0 0 0 2: 2 4 12 | 0 0 0 3: 1 2 14 | 0 0 0 4: 1 2 16 | 0 0 0 5: 1 2 18 | 0 0 0 6: 1 2 20 | 0 0 0 7: 1 2 22 | 0 0 0 8: 2 4 26 | 0 0 0 9: 37 74 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 osc.lustre-OST0000-osc-ffff8800aae60000.rpc_stats= snapshot_time: 1746072107.450673441 secs.nsecs start_time: 1746071564.479071673 secs.nsecs elapsed_time: 542.971601768 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 3 0 0 | 3 1 1 2: 0 0 0 | 0 0 1 4: 0 0 0 | 0 0 1 8: 0 0 0 | 0 0 1 16: 0 0 0 | 0 0 1 32: 0 0 0 | 0 0 1 64: 1 0 1 | 0 0 1 128: 1 0 1 | 0 0 1 256: 352 98 100 | 150 98 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 38 10 10 | 14 9 9 2: 25 7 17 | 67 43 52 3: 18 5 22 | 54 35 88 4: 19 5 28 | 17 11 99 5: 15 4 32 | 1 0 100 6: 15 4 36 | 0 0 100 7: 14 3 40 | 0 0 100 8: 20 5 45 | 0 0 100 9: 192 53 99 | 0 0 100 10: 1 0 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 12 3 3 | 6 3 3 1: 0 0 3 | 0 0 3 2: 0 0 3 | 0 0 3 4: 0 0 3 | 0 0 3 8: 0 0 3 | 0 0 3 16: 0 0 3 | 0 0 3 32: 0 0 3 | 0 0 3 64: 0 0 3 | 0 0 3 128: 0 0 3 | 0 0 3 256: 7 1 5 | 3 1 5 512: 14 3 9 | 6 3 9 1024: 28 7 17 | 12 7 17 2048: 57 15 33 | 24 15 33 4096: 113 31 64 | 48 31 64 8192: 126 35 100 | 54 35 100 osc.lustre-OST0001-osc-ffff8800aadd2000.rpc_stats= snapshot_time: 1746072107.450745460 secs.nsecs start_time: 1746071566.333534049 secs.nsecs elapsed_time: 541.117211411 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 5 10 10 | 0 0 0 2: 3 6 16 | 0 0 0 3: 2 4 20 | 0 0 0 4: 2 4 24 | 0 0 0 5: 1 2 26 | 0 0 0 6: 1 2 28 | 0 0 0 7: 1 2 30 | 0 0 0 8: 1 2 32 | 0 0 0 9: 34 68 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 osc.lustre-OST0001-osc-ffff8800aae60000.rpc_stats= snapshot_time: 1746072107.450806486 secs.nsecs start_time: 1746071564.480019807 secs.nsecs elapsed_time: 542.970786679 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 1223 6 6 | 3 0 0 2: 11 0 6 | 0 0 0 4: 8 0 6 | 0 0 0 8: 10 0 6 | 0 0 0 16: 3875 20 27 | 0 0 0 32: 445 2 30 | 0 0 0 64: 558 3 33 | 0 0 0 128: 1341 7 40 | 0 0 0 256: 11005 59 100 | 3000 99 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 10646 57 57 | 278 9 9 2: 1809 9 67 | 703 23 32 3: 884 4 72 | 731 24 57 4: 681 3 75 | 493 16 73 5: 648 3 79 | 349 11 85 6: 513 2 82 | 186 6 91 7: 481 2 84 | 102 3 94 8: 472 2 87 | 72 2 97 9: 2238 12 99 | 87 2 99 10: 18 0 99 | 2 0 100 11: 9 0 99 | 0 0 100 12: 8 0 99 | 0 0 100 13: 8 0 99 | 0 0 100 14: 8 0 99 | 0 0 100 15: 6 0 99 | 0 0 100 16: 5 0 99 | 0 0 100 17: 3 0 99 | 0 0 100 18: 3 0 99 | 0 0 100 19: 3 0 99 | 0 0 100 20: 3 0 99 | 0 0 100 21: 3 0 99 | 0 0 100 22: 3 0 99 | 0 0 100 23: 3 0 99 | 0 0 100 24: 3 0 99 | 0 0 100 25: 2 0 99 | 0 0 100 26: 2 0 99 | 0 0 100 27: 2 0 99 | 0 0 100 28: 2 0 99 | 0 0 100 29: 2 0 99 | 0 0 100 30: 2 0 99 | 0 0 100 31: 6 0 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 225 1 1 | 63 2 2 1: 74 0 1 | 0 0 2 2: 140 0 2 | 0 0 2 4: 3 0 2 | 0 0 2 8: 0 0 2 | 0 0 2 16: 132 0 3 | 0 0 2 32: 234 1 4 | 0 0 2 64: 394 2 6 | 0 0 2 128: 793 4 10 | 0 0 2 256: 413 2 13 | 60 1 4 512: 769 4 17 | 120 3 8 1024: 1516 8 25 | 240 7 16 2048: 2775 15 40 | 480 15 32 4096: 5174 28 68 | 960 31 64 8192: 5834 31 100 | 1080 35 100 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d99.sanity-pcc is idle now PASS 99 (69s) == sanity-pcc test 99b: race among unlink | two readers | write | detach for PCC-RO file ========================================================== 00:01:51 (1746072111) 200+0 records in 200+0 records out 209715200 bytes (210 MB) copied, 0.402164 s, 521 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d99b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/204800 done Creating filesystem with 204800 1k blocks and 51200 inodes Filesystem UUID: 94b0e37e-52ad-4a12-9b6f-19f8630c4826 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/25 done Writing inode tables: 0/25 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/25 done /tmp/f99b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=94b0e37e-52ad-4a12-9b6f-19f8630c4826 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.538793 s, 97.3 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.694325 s, 75.5 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.02518 s, 51.1 MB/s unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563967 s, 930 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.59576 s, 88.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.725776 s, 72.2 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259798 s, 2.0 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.564244 s, 92.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.603814 s, 86.8 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.842992 s, 62.2 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.660323 s, 79.4 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.567792 s, 92.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408031 s, 1.3 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.352167 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.658915 s, 79.6 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450171 s, 1.2 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.406172 s, 129 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237702 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.656579 s, 79.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275426 s, 1.9 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.382948 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.747269 s, 70.2 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404875 s, 1.3 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.614092 s, 85.4 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.652746 s, 80.3 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342825 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.53083 s, 98.8 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.469046 s, 112 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.599629 s, 87.4 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043822 s, 1.2 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.351425 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.669202 s, 78.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243978 s, 2.1 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.717356 s, 73.1 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.663193 s, 79.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433444 s, 1.2 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.995048 s, 52.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.416723 s, 126 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0976518 s, 537 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.633295 s, 82.8 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.623069 s, 84.1 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.600278 s, 87.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.858115 s, 61.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.192215 s, 273 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314005 s, 1.7 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.578941 s, 90.6 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.67308 s, 77.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.582681 s, 90.0 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302211 s, 1.7 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.661628 s, 79.2 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.398036 s, 132 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114115 s, 459 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.715964 s, 73.2 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.456135 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.267116 s, 196 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.11007 s, 47.2 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.851424 s, 61.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.585468 s, 89.6 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365994 s, 1.4 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.433541 s, 121 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.677976 s, 77.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.818975 s, 64.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317412 s, 1.7 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.739169 s, 70.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.509077 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498169 s, 1.1 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.807547 s, 64.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.428999 s, 122 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655542 s, 800 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.762031 s, 68.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.905262 s, 57.9 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.735619 s, 71.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427591 s, 1.2 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.817126 s, 64.2 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.55479 s, 94.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300459 s, 1.7 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.887884 s, 59.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034363 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103523 s, 506 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.267229 s, 196 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.387275 s, 135 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.74124 s, 70.7 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.592783 s, 88.4 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424566 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.543254 s, 96.5 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.668871 s, 78.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.667361 s, 78.6 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631208 s, 831 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.606691 s, 86.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.464728 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.230562 s, 227 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.613157 s, 85.5 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.527921 s, 99.3 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.623612 s, 84.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.939462 s, 55.8 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666895 s, 786 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240547 s, 2.2 GB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.449704 s, 117 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.542255 s, 96.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369086 s, 1.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.688955 s, 76.1 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.462641 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435836 s, 1.2 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.661378 s, 79.3 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.605073 s, 86.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196041 s, 267 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.792775 s, 66.1 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.580256 s, 90.4 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.928879 s, 56.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.0322 s, 50.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321486 s, 1.6 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.739351 s, 70.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.6257 s, 83.8 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.340511 s, 154 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.730568 s, 71.8 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.961863 s, 54.5 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414622 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.931528 s, 56.3 MB/s unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.472252 s, 111 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.4948 s, 106 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.674569 s, 77.7 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657909 s, 797 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648519 s, 808 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.610372 s, 85.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.38478 s, 37.9 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.985125 s, 53.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.836215 s, 62.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545481 s, 961 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cd/0x2000059f1:0x3c:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... ==== DONE ==== 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.23517 s, 42.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.11433 s, 47.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543828 s, 964 MB/s ==== DONE WIAT ==== osc.lustre-OST0000-osc-ffff8800aadd2000.rpc_stats= snapshot_time: 1746072174.607331738 secs.nsecs start_time: 1746071566.333139792 secs.nsecs elapsed_time: 608.274191946 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 4 8 8 | 0 0 0 2: 2 4 12 | 0 0 0 3: 1 2 14 | 0 0 0 4: 1 2 16 | 0 0 0 5: 1 2 18 | 0 0 0 6: 1 2 20 | 0 0 0 7: 1 2 22 | 0 0 0 8: 2 4 26 | 0 0 0 9: 37 74 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 osc.lustre-OST0000-osc-ffff8800aae60000.rpc_stats= snapshot_time: 1746072174.607474006 secs.nsecs start_time: 1746071564.479071673 secs.nsecs elapsed_time: 610.128402333 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 3 0 0 | 3 1 1 2: 0 0 0 | 0 0 1 4: 0 0 0 | 0 0 1 8: 0 0 0 | 0 0 1 16: 0 0 0 | 0 0 1 32: 0 0 0 | 0 0 1 64: 1 0 1 | 0 0 1 128: 1 0 1 | 0 0 1 256: 352 98 100 | 150 98 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 38 10 10 | 14 9 9 2: 25 7 17 | 67 43 52 3: 18 5 22 | 54 35 88 4: 19 5 28 | 17 11 99 5: 15 4 32 | 1 0 100 6: 15 4 36 | 0 0 100 7: 14 3 40 | 0 0 100 8: 20 5 45 | 0 0 100 9: 192 53 99 | 0 0 100 10: 1 0 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 12 3 3 | 6 3 3 1: 0 0 3 | 0 0 3 2: 0 0 3 | 0 0 3 4: 0 0 3 | 0 0 3 8: 0 0 3 | 0 0 3 16: 0 0 3 | 0 0 3 32: 0 0 3 | 0 0 3 64: 0 0 3 | 0 0 3 128: 0 0 3 | 0 0 3 256: 7 1 5 | 3 1 5 512: 14 3 9 | 6 3 9 1024: 28 7 17 | 12 7 17 2048: 57 15 33 | 24 15 33 4096: 113 31 64 | 48 31 64 8192: 126 35 100 | 54 35 100 osc.lustre-OST0001-osc-ffff8800aadd2000.rpc_stats= snapshot_time: 1746072174.607539730 secs.nsecs start_time: 1746071566.333534049 secs.nsecs elapsed_time: 608.274005681 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 5 10 10 | 0 0 0 2: 3 6 16 | 0 0 0 3: 2 4 20 | 0 0 0 4: 2 4 24 | 0 0 0 5: 1 2 26 | 0 0 0 6: 1 2 28 | 0 0 0 7: 1 2 30 | 0 0 0 8: 1 2 32 | 0 0 0 9: 34 68 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 osc.lustre-OST0001-osc-ffff8800aae60000.rpc_stats= snapshot_time: 1746072174.607609456 secs.nsecs start_time: 1746071564.480019807 secs.nsecs elapsed_time: 610.127589649 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 1223 5 5 | 3 0 0 2: 11 0 5 | 0 0 0 4: 8 0 5 | 0 0 0 8: 10 0 5 | 0 0 0 16: 3875 18 24 | 0 0 0 32: 445 2 26 | 0 0 0 64: 558 2 28 | 0 0 0 128: 1341 6 35 | 0 0 0 256: 13755 64 100 | 5350 99 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 11130 52 52 | 473 8 8 2: 2139 10 62 | 1244 23 32 3: 1128 5 67 | 1301 24 56 4: 939 4 72 | 922 17 73 5: 879 4 76 | 617 11 85 6: 726 3 79 | 343 6 91 7: 669 3 82 | 183 3 94 8: 673 3 86 | 121 2 97 9: 2838 13 99 | 147 2 99 10: 19 0 99 | 2 0 100 11: 9 0 99 | 0 0 100 12: 8 0 99 | 0 0 100 13: 8 0 99 | 0 0 100 14: 8 0 99 | 0 0 100 15: 6 0 99 | 0 0 100 16: 5 0 99 | 0 0 100 17: 3 0 99 | 0 0 100 18: 3 0 99 | 0 0 100 19: 3 0 99 | 0 0 100 20: 3 0 99 | 0 0 100 21: 3 0 99 | 0 0 100 22: 3 0 99 | 0 0 100 23: 3 0 99 | 0 0 100 24: 3 0 99 | 0 0 100 25: 2 0 99 | 0 0 100 26: 2 0 99 | 0 0 100 27: 2 0 99 | 0 0 100 28: 2 0 99 | 0 0 100 29: 2 0 99 | 0 0 100 30: 2 0 99 | 0 0 100 31: 6 0 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 293 1 1 | 110 2 2 1: 74 0 1 | 0 0 2 2: 140 0 2 | 0 0 2 4: 3 0 2 | 0 0 2 8: 0 0 2 | 0 0 2 16: 132 0 3 | 0 0 2 32: 234 1 4 | 0 0 2 64: 394 1 5 | 0 0 2 128: 793 3 9 | 0 0 2 256: 476 2 11 | 107 1 4 512: 893 4 16 | 214 3 8 1024: 1759 8 24 | 428 7 16 2048: 3244 15 39 | 856 15 32 4096: 6008 28 68 | 1712 31 64 8192: 6783 31 100 | 1926 35 100 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d99b.sanity-pcc is idle now PASS 99b (66s) == sanity-pcc test 100: race among PCC unlink | read | write | detach for PCC-RO file ========================================================== 00:02:57 (1746072177) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0996094 s, 632 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d100.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 8f27827d-39e7-44a3-adf4-04dd8ae4e413 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f100.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=8f27827d-39e7-44a3-adf4-04dd8ae4e413 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.564639 s, 92.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.602677 s, 7.0 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0333098 s, 31.5 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 45+0 records in 45+0 records out 47185920 bytes (47 MB) copied, 0.196245 s, 240 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.795587 s, 65.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584807 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425073 s, 1.2 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00364715 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0171096 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.808016 s, 64.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.478607 s, 110 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189854 s, 2.8 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.452005 s, 116 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 39+0 records in 39+0 records out 40894464 bytes (41 MB) copied, 0.147854 s, 277 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.786984 s, 66.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.571301 s, 91.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.840536 s, 62.4 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0950715 s, 11.0 MB/s 22+0 records in 22+0 records out 23068672 bytes (23 MB) copied, 0.052429 s, 440 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.620716 s, 84.5 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.494794 s, 106 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394843 s, 1.3 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.589035 s, 89.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.907252 s, 57.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.397326 s, 132 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00527419 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0101148 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.000377164 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.000289633 s, 0.0 kB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.628822 s, 83.4 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.000293778 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00231507 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.632837 s, 82.8 MB/s 14+0 records in 14+0 records out 14680064 bytes (15 MB) copied, 0.293279 s, 50.1 MB/s 14+0 records in 14+0 records out 14680064 bytes (15 MB) copied, 0.29908 s, 49.1 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.709261 s, 73.9 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.461949 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403357 s, 1.3 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0162108 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.343741 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.530215 s, 98.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.02607 s, 51.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.356037 s, 5.9 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.015912 s, 65.9 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.654241 s, 80.1 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.495702 s, 106 MB/s 45+0 records in 45+0 records out 47185920 bytes (47 MB) copied, 0.229418 s, 206 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.549022 s, 95.5 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.473117 s, 111 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387524 s, 1.4 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.631172 s, 83.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.264544 s, 198 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.433723 s, 121 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.608025 s, 86.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.383462 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284324 s, 1.8 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0187299 s, 0.0 kB/s 37+0 records in 37+0 records out 38797312 bytes (39 MB) copied, 0.160599 s, 242 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.832314 s, 63.0 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 34+0 records in 34+0 records out 35651584 bytes (36 MB) copied, 0.466884 s, 76.4 MB/s 34+0 records in 34+0 records out 35651584 bytes (36 MB) copied, 0.59261 s, 60.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.595231 s, 88.1 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.528861 s, 99.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487375 s, 1.1 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.00905794 s, 116 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.886816 s, 59.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.649071 s, 80.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.305414 s, 172 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 9+0 records in 9+0 records out 9437184 bytes (9.4 MB) copied, 0.482566 s, 19.6 MB/s 3+0 records in 3+0 records out 3145728 bytes (3.1 MB) copied, 0.017026 s, 185 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.782148 s, 67.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.415228 s, 126 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293115 s, 1.8 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0179585 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.534081 s, 98.2 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.061735 s, 17.0 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0197378 s, 53.1 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.59586 s, 88.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.640417 s, 81.9 MB/s 31+0 records in 31+0 records out 32505856 bytes (33 MB) copied, 0.153675 s, 212 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.234 s, 67.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.968173 s, 54.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.801476 s, 65.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.369145 s, 142 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.734369 s, 71.4 MB/s 22+0 records in 22+0 records out 23068672 bytes (23 MB) copied, 0.548481 s, 42.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 3+0 records in 3+0 records out 3145728 bytes (3.1 MB) copied, 0.0417562 s, 75.3 MB/s unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory 44+0 records in 44+0 records out 46137344 bytes (46 MB) copied, 0.177836 s, 259 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.583026 s, 89.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 14+1 records in 14+1 records out 14983168 bytes (15 MB) copied, 0.0731554 s, 205 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0287454 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.642456 s, 81.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.264292 s, 198 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443266 s, 1.2 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0139459 s, 75.2 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 34+0 records in 34+0 records out 35651584 bytes (36 MB) copied, 0.0442881 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.543006 s, 96.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022544 s, 2.3 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 45+0 records in 45+0 records out 47185920 bytes (47 MB) copied, 0.284868 s, 166 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0822009 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.812272 s, 64.5 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.798184 s, 65.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.295018 s, 178 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.572772 s, 91.5 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.427259 s, 123 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 23+0 records in 23+0 records out 24117248 bytes (24 MB) copied, 0.123618 s, 195 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0147892 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 43+0 records in 43+0 records out 45088768 bytes (45 MB) copied, 0.334619 s, 135 MB/s 43+0 records in 43+0 records out 45088768 bytes (45 MB) copied, 0.202934 s, 222 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.687909 s, 76.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 17+0 records in 17+0 records out 17825792 bytes (18 MB) copied, 0.328996 s, 54.2 MB/s 3+0 records in 3+0 records out 3145728 bytes (3.1 MB) copied, 0.0156002 s, 202 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 48+0 records in 48+0 records out 50331648 bytes (50 MB) copied, 0.210419 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.694042 s, 75.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243998 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272739 s, 1.9 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.49199 s, 107 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.350906 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134098 s, 391 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00531801 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 37+0 records in 37+0 records out 38797312 bytes (39 MB) copied, 0.258907 s, 150 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.579593 s, 90.5 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522896 s, 1.0 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0236816 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.88903 s, 59.0 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.412746 s, 127 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107817 s, 486 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00393507 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.468169 s, 112 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372394 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231887 s, 2.3 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0102115 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 40+0 records in 40+0 records out 41943040 bytes (42 MB) copied, 0.157995 s, 265 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.577071 s, 90.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.402169 s, 130 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293692 s, 1.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 48+0 records in 48+0 records out 50331648 bytes (50 MB) copied, 0.323509 s, 156 MB/s 48+0 records in 48+0 records out 50331648 bytes (50 MB) copied, 0.201289 s, 250 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.689339 s, 76.1 MB/s 6+0 records in 6+0 records out 6291456 bytes (6.3 MB) copied, 0.375838 s, 16.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0137288 s, 76.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.443824 s, 118 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.296339 s, 177 MB/s 14+0 records in 14+0 records out 14680064 bytes (15 MB) copied, 0.0235585 s, 623 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.513564 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726251 s, 722 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250355 s, 2.1 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0112003 s, 0.0 kB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.42128 s, 124 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.326143 s, 161 MB/s 48+0 records in 48+0 records out 50331648 bytes (50 MB) copied, 0.126366 s, 398 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00918778 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.50016 s, 105 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.362085 s, 145 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 30+0 records in 30+0 records out 31457280 bytes (31 MB) copied, 0.119453 s, 263 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0149053 s, 0.0 kB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.492475 s, 106 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252025 s, 2.1 GB/s 24+0 records in 24+0 records out 25165824 bytes (25 MB) copied, 0.0986246 s, 255 MB/s 22+0 records in 22+0 records out 23068672 bytes (23 MB) copied, 0.0803299 s, 287 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.489577 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.208199 s, 252 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271614 s, 1.9 GB/s 49+0 records in 49+0 records out 51380224 bytes (51 MB) copied, 0.114214 s, 450 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.612438 s, 85.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.369261 s, 142 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0569866 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.556903 s, 94.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.458912 s, 114 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0223254 s, 47.0 MB/s 48+0 records in 48+0 records out 50331648 bytes (50 MB) copied, 0.254717 s, 198 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.567497 s, 92.4 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.359399 s, 146 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228337 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.532464 s, 98.5 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.523488 s, 100 MB/s 43+0 records in 43+0 records out 45088768 bytes (45 MB) copied, 0.127518 s, 354 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a6/0x2000059f1:0x57:0x0': No such file or directory 31+0 records in 31+0 records out 32505856 bytes (33 MB) copied, 0.111141 s, 292 MB/s 31+0 records in 31+0 records out 32505856 bytes (33 MB) copied, 0.0507651 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.637709 s, 82.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 42+0 records in 42+0 records out 44040192 bytes (44 MB) copied, 0.149665 s, 294 MB/s 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.0141151 s, 297 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.38577 s, 136 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d100.sanity-pcc is idle now PASS 100 (65s) == sanity-pcc test 101a: Test auto attach in mount namespace (simulated container) ========================================================== 00:04:02 (1746072242) creating user namespace for 500 sanityu+ 19051 0.0 0.0 108060 364 pts/0 S+ 00:04 0:00 sleep 600 root 19053 0.0 0.0 114880 984 pts/0 S+ 00:04 0:00 grep sleep Created NS: child (sleep) pid=19051 in 0 seconds running as uid/gid/euid/egid 500/500/500/500, groups: 500 [newuidmap] [19051] [0] [500] [1] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [newgidmap] [19051] [0] [500] [1] 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069041 s, 759 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d101a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 9f635a4b-0b3d-4aab-af77-7287068ada9a Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f101a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=9f635a4b-0b3d-4aab-af77-7287068ada9a (extents) (64bit) (large files) (huge files) Check for temp file in PCC mount Check for temp file in PCC mount from inside namespace Starting copytool 'agt1' on 'oleg311-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d101a.sanity-pcc/d101a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Verify open attach from inside mount namespace 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0373533 s, 27.4 kB/s file: /mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc, type: readwrite, PCC_file: /d101a.sanity-pcc/5983/0x2000059f1:0x72:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] ldlm.namespaces.lustre-MDT0000-mdc-ffff8800aadd2000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800aae60000.lru_size=clear running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0020357 s, 503 kB/s Verify auto attach at open from inside NS for RW-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800aadd2000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800aae60000.lru_size=clear Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d101a.sanity-pcc is idle now PASS 101a (9s) == sanity-pcc test 102: PCC-RO should not hange for io_uring I/O engine ========================================================== 00:04:11 (1746072251) SKIP: sanity-pcc test_102 Client OS does not support io_uring I/O engine SKIP 102 (2s) == sanity-pcc test 203: Verify attach/hit bytes statistics data ========================================================== 00:04:13 (1746072253) 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.0555843 s, 189 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d203.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/10240 done Creating filesystem with 10240 1k blocks and 2560 inodes Filesystem UUID: b538e4b6-e918-4b5b-a750-f91f39ecf7ee Superblock backups stored on blocks: 8193 Allocating group tables: 0/2 done Writing inode tables: 0/2 done Creating journal (2048 blocks): done Writing superblocks and filesystem accounting information: 0/2 done /tmp/f203.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b538e4b6-e918-4b5b-a750-f91f39ecf7ee (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d203.sanity-pcc/d203.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00557219 s, 184 kB/s ==== stats ==== llite.lustre-ffff8800aadd2000.stats= snapshot_time 1746072254.582591230 secs.nsecs start_time 1746072254.497952544 secs.nsecs elapsed_time 0.084638686 secs.nsecs llite.lustre-ffff8800aae60000.stats= snapshot_time 1746072254.582680809 secs.nsecs start_time 1746072254.497969741 secs.nsecs elapsed_time 0.084711068 secs.nsecs read_bytes 1 samples [bytes] 1024 1024 1024 1048576 write_bytes 1 samples [bytes] 1024 1024 1024 1048576 read 3 samples [usecs] 127 5453 5740 29776938 write 1 samples [usecs] 2960 2960 2960 8761600 open 3 samples [usecs] 17 24780 24818 614049130 close 3 samples [usecs] 2233 10986 16184 134469710 getattr 2 samples [usecs] 259 8519 8778 72640442 mknod 1 samples [usecs] 5947 5947 5947 35366809 inode_permission 7 samples [usecs] 2 2593 2815 6741329 opencount 3 samples [reqs] 1 3 6 14 openclosetime 2 samples [usecs] 8966 21280 30246 533227556 pcc_attach 1 samples [reqs] pcc_hit_bytes 2 samples [bytes] 1024 1024 2048 2097152 pcc_attach_bytes 1 samples [bytes] 1024 1024 1024 1048576 attach_num: 1, attach_bytes: 1024, hit_num: 2, hit_bytes: 2048 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d203.sanity-pcc is idle now PASS 203 (4s) == sanity-pcc test 204a: pin/unpin pcc flag ============== 00:04:18 (1746072258) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811605 s, 646 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d204a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 365cf8d3-7b87-4f5d-a2eb-521789ba5414 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f204a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=365cf8d3-7b87-4f5d-a2eb-521789ba5414 (extents) (64bit) (large files) (huge files) getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2,hsm: 100]" getfattr: Removing leading '/' from absolute path names getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2,hsm: 100]" getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names Cleanup PCC backend on /mnt/lustre /mnt/pcc.d204a.sanity-pcc is idle now PASS 204a (4s) == sanity-pcc test 204b: Permission check for the pin/unpin operation ========================================================== 00:04:23 (1746072263) mdt.lustre-MDT0000.enable_pin_gid=0 /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh: illegal option -- p running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] pin: cannot pin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc trusted.pin="[hsm: 100]" running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] unpin: cannot unpin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] pin: cannot pin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted mdt.lustre-MDT0000.enable_pin_gid=-1 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] mdt.lustre-MDT0000.enable_pin_gid=500 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] mdt.lustre-MDT0000.enable_pin_gid=0 PASS 204b (5s) == sanity-pcc test 204c: PCC pin/unpin without attach ID specified ========================================================== 00:04:28 (1746072268) pin: failed to get id for '/mnt/lustre/f204c.sanity-pcc': No such file or directory 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.106583 s, 590 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d204c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12248) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg311-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg311-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51243/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.11,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.111@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.111@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: d0918ae1-cef2-4f5e-90a4-1c6961a6e3d3 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f204c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d0918ae1-cef2-4f5e-90a4-1c6961a6e3d3 (extents) (64bit) (large files) (huge files) getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204c.sanity-pcc trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names /mnt/lustre/f204c.sanity-pcc: trusted.pin: No such attribute /mnt/lustre/f204c.sanity-pcc: trusted.pin: No such attribute Cleanup PCC backend on /mnt/lustre /mnt/pcc.d204c.sanity-pcc is idle now PASS 204c (5s) == sanity-pcc test complete, duration 1827 sec =========== 00:04:33 (1746072273) === sanity-pcc: start cleanup 00:04:33 (1746072273) === Stopping clients: oleg311-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg311-client.virtnet /mnt/lustre2 opts: === sanity-pcc: finish cleanup 00:04:37 (1746072277) ===