== sanity test 123ad: Verify batching statahead works correctly ========================================================== 15:00:09 (1788807609) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.batch_stats=0 llite.lustre-ffff8d1d91ffd000.statahead_max=32 llite.lustre-ffff8d1d91ffd000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.90 seconds: 34.45 ops/second llite.lustre-ffff8d1d91ffd000.statahead_max=0 101 real 0m2.250s user 0m0.003s sys 0m0.322s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff8d1d91ffd000.statahead_max=32 llite.lustre-ffff8d1d91ffd000.statahead_max=32 101 real 0m1.175s user 0m0.002s sys 0m0.187s 'ls -l' 100 files with statahead: 1 sec statahead total: 123 statahead wrong: 2 agl total: 118 list_total: 30 fname_total: 93 hit_total: 3918 miss_total: 1550 swrong=2 ewrong=2 - open/close 813 (time 1788807637.89 total 10.00 last 81.27) total: 900 open/close in 11.28 seconds: 79.77 ops/second llite.lustre-ffff8d1d91ffd000.statahead_max=0 1001 real 0m21.573s user 0m0.033s sys 0m3.015s 'ls -l' 1000 files without statahead: 22 sec llite.lustre-ffff8d1d91ffd000.statahead_max=32 llite.lustre-ffff8d1d91ffd000.statahead_max=32 1001 real 0m10.501s user 0m0.037s sys 0m1.520s 'ls -l' 1000 files with statahead: 11 sec statahead total: 124 statahead wrong: 2 agl total: 119 list_total: 31 fname_total: 93 hit_total: 4916 miss_total: 1552 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 21 seconds llite.lustre-ffff8d1d91ffd000.statahead_stats= statahead total: 124 statahead wrong: 2 agl total: 119 list_total: 31 fname_total: 93 hit_total: 4916 miss_total: 1552 mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.batch_stats= snapshot_time: 1788807707.334379759 secs.nsecs start_time: 1788807614.655641866 secs.nsecs elapsed_time: 92.678737893 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 3 8 8 8: 3 8 17 16: 23 65 82 32: 6 17 100 mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.batch_stats= snapshot_time: 1788807707.334453258 secs.nsecs start_time: 1788807614.658317941 secs.nsecs elapsed_time: 92.676135317 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 1 2 5 4: 1 2 7 8: 2 5 12 16: 10 25 37 32: 25 62 100 - open/close 688 (time 1788807720.03 total 10.01 last 68.74) - open/close 1411 (time 1788807730.03 total 20.02 last 72.25) - open/close 2168 (time 1788807740.05 total 30.03 last 75.56) - open/close 2844 (time 1788807750.07 total 40.05 last 67.50) - open/close 3516 (time 1788807760.07 total 50.05 last 67.20) - open/close 4241 (time 1788807770.08 total 60.06 last 72.42) - open/close 4994 (time 1788807780.08 total 70.07 last 75.27) - open/close 5686 (time 1788807790.09 total 80.07 last 69.18) - open/close 6424 (time 1788807800.10 total 90.08 last 73.73) - open/close 7169 (time 1788807810.10 total 100.08 last 74.45) - open/close 7902 (time 1788807820.11 total 110.09 last 73.23) - open/close 8542 (time 1788807830.14 total 120.12 last 63.86) - open/close 9276 (time 1788807840.14 total 130.12 last 73.38) total: 10000 open/close in 139.87 seconds: 71.49 ops/second llite.lustre-ffff8d1d91ffd000.statahead_batch_max=0 llite.lustre-ffff8d1d91ffd000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.stats=clear mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m59.766s user 0m0.342s sys 0m15.632s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8d1d91ffd000.statahead_batch_max=32 llite.lustre-ffff8d1d91ffd000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.stats=clear mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m36.570s user 0m0.373s sys 0m14.983s batched statahead RPC= hit count=9980 mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.batch_stats= snapshot_time: 1788808024.542640769 secs.nsecs start_time: 1788807918.303574285 secs.nsecs elapsed_time: 106.239066484 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.batch_stats= snapshot_time: 1788808024.542840680 secs.nsecs start_time: 1788807918.303712355 secs.nsecs elapsed_time: 106.239128325 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.batch_stats=0 llite.lustre-ffff8d1d91ffd000.statahead_max=2048 llite.lustre-ffff8d1d91ffd000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.31 seconds: 43.35 ops/second llite.lustre-ffff8d1d91ffd000.statahead_max=0 101 real 0m1.833s user 0m0.004s sys 0m0.301s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff8d1d91ffd000.statahead_max=2048 llite.lustre-ffff8d1d91ffd000.statahead_max=2048 101 real 0m0.938s user 0m0.006s sys 0m0.162s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10079 miss_total: 21 swrong=0 ewrong=0 - open/close 764 (time 1788808313.87 total 10.01 last 76.32) total: 900 open/close in 11.79 seconds: 76.34 ops/second llite.lustre-ffff8d1d91ffd000.statahead_max=0 1001 real 0m19.288s user 0m0.051s sys 0m3.004s 'ls -l' 1000 files without statahead: 20 sec llite.lustre-ffff8d1d91ffd000.statahead_max=2048 llite.lustre-ffff8d1d91ffd000.statahead_max=2048 1001 real 0m5.223s user 0m0.017s sys 0m1.397s 'ls -l' 1000 files with statahead: 5 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11078 miss_total: 22 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 20 seconds llite.lustre-ffff8d1d91ffd000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11078 miss_total: 22 mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.batch_stats= snapshot_time: 1788808370.991306933 secs.nsecs start_time: 1788808026.937052770 secs.nsecs elapsed_time: 344.054254163 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 3 33 33 8: 1 11 44 16: 1 11 55 32: 2 22 77 64: 0 0 77 128: 0 0 77 256: 2 22 100 mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.batch_stats= snapshot_time: 1788808370.991415969 secs.nsecs start_time: 1788808026.938073857 secs.nsecs elapsed_time: 344.053342112 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 22 22 8: 2 22 44 16: 0 0 44 32: 3 33 77 64: 0 0 77 128: 0 0 77 256: 2 22 100 - open/close 843 (time 1788808383.83 total 10.00 last 84.26) - open/close 1673 (time 1788808393.84 total 20.02 last 82.88) - open/close 2490 (time 1788808403.85 total 30.02 last 81.68) - open/close 3334 (time 1788808413.86 total 40.03 last 84.30) - open/close 4109 (time 1788808423.86 total 50.04 last 77.46) - open/close 5003 (time 1788808433.87 total 60.04 last 89.34) - open/close 5985 (time 1788808443.88 total 70.05 last 98.12) - open/close 6893 (time 1788808453.88 total 80.06 last 90.75) - open/close 7807 (time 1788808463.90 total 90.08 last 91.22) - open/close 8737 (time 1788808473.91 total 100.08 last 92.98) - open/close 9652 (time 1788808483.91 total 110.09 last 91.45) total: 10000 open/close in 113.75 seconds: 87.91 ops/second llite.lustre-ffff8d1d91ffd000.statahead_batch_max=0 llite.lustre-ffff8d1d91ffd000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.stats=clear mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m43.568s user 0m0.283s sys 0m16.139s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8d1d91ffd000.statahead_batch_max=256 llite.lustre-ffff8d1d91ffd000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.stats=clear mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m37.193s user 0m0.271s sys 0m15.005s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff8d1d91ffd000.batch_stats= snapshot_time: 1788808581.438604435 secs.nsecs start_time: 1788808537.263717812 secs.nsecs elapsed_time: 44.174886623 secs.nsecs subreqs per batch batches %% cum %% 1: 92 44 44 2: 27 12 56 4: 14 6 63 8: 10 4 68 16: 8 3 72 32: 17 8 80 64: 5 2 82 128: 1 0 83 256: 35 16 100 mdc.lustre-MDT0001-mdc-ffff8d1d91ffd000.batch_stats= snapshot_time: 1788808581.438829353 secs.nsecs start_time: 1788808537.263862219 secs.nsecs elapsed_time: 44.174967134 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff8d1d91ffd000.statahead_batch_max=64 llite.lustre-ffff8d1d91ffd000.statahead_max=128