== sanity test 123ad: Verify batching statahead works correctly ========================================================== 02:42:40 (1789022560) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.batch_stats=0 llite.lustre-ffff9c1c993d3000.statahead_max=32 llite.lustre-ffff9c1c993d3000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.batch_stats=0 striped dir -i1 -c2 -H all_char /mnt/lustre/d123ad.sanity total: 100 open/close in 2.04 seconds: 49.08 ops/second llite.lustre-ffff9c1c993d3000.statahead_max=0 101 real 0m2.738s user 0m0.007s sys 0m0.308s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff9c1c993d3000.statahead_max=32 llite.lustre-ffff9c1c993d3000.statahead_max=32 101 real 0m0.990s user 0m0.004s sys 0m0.169s 'ls -l' 100 files with statahead: 1 sec statahead total: 147 statahead wrong: 2 agl total: 141 list_total: 31 fname_total: 87 hit_total: 3888 miss_total: 1675 swrong=2 ewrong=2 - open/close 671 (time 1789022588.36 total 10.00 last 67.08) total: 900 open/close in 12.93 seconds: 69.62 ops/second llite.lustre-ffff9c1c993d3000.statahead_max=0 1001 real 0m28.839s user 0m0.030s sys 0m2.901s 'ls -l' 1000 files without statahead: 29 sec llite.lustre-ffff9c1c993d3000.statahead_max=32 llite.lustre-ffff9c1c993d3000.statahead_max=32 1001 real 0m11.874s user 0m0.028s sys 0m1.343s 'ls -l' 1000 files with statahead: 11 sec statahead total: 148 statahead wrong: 2 agl total: 142 list_total: 32 fname_total: 87 hit_total: 4885 miss_total: 1678 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 19 seconds llite.lustre-ffff9c1c993d3000.statahead_stats= statahead total: 148 statahead wrong: 2 agl total: 142 list_total: 32 fname_total: 87 hit_total: 4885 miss_total: 1678 mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.batch_stats= snapshot_time: 1789022667.994913701 secs.nsecs start_time: 1789022566.062853505 secs.nsecs elapsed_time: 101.932060196 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 3 7 7 8: 3 7 15 16: 18 46 61 32: 15 38 100 mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.batch_stats= snapshot_time: 1789022667.994984206 secs.nsecs start_time: 1789022566.062927835 secs.nsecs elapsed_time: 101.932056371 secs.nsecs subreqs per batch batches %% cum %% 1: 2 5 5 2: 0 0 5 4: 2 5 10 8: 2 5 15 16: 21 52 67 32: 13 32 100 - open/close 773 (time 1789022680.54 total 10.00 last 77.30) - open/close 1588 (time 1789022690.55 total 20.01 last 81.42) - open/close 2585 (time 1789022700.55 total 30.01 last 99.69) - open/close 3567 (time 1789022710.56 total 40.02 last 98.14) - open/close 4406 (time 1789022720.56 total 50.02 last 83.88) - open/close 5313 (time 1789022730.57 total 60.03 last 90.60) - open/close 6259 (time 1789022740.58 total 70.04 last 94.56) - open/close 6992 (time 1789022750.60 total 80.05 last 73.16) - open/close 7820 (time 1789022760.63 total 90.08 last 82.57) - open/close 8695 (time 1789022770.64 total 100.09 last 87.41) - open/close 9569 (time 1789022780.65 total 110.11 last 87.28) total: 10000 open/close in 114.98 seconds: 86.97 ops/second llite.lustre-ffff9c1c993d3000.statahead_batch_max=0 llite.lustre-ffff9c1c993d3000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.stats=clear mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m7.784s user 0m0.306s sys 0m14.334s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9c1c993d3000.statahead_batch_max=32 llite.lustre-ffff9c1c993d3000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.stats=clear mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m48.193s user 0m0.326s sys 0m13.466s batched statahead RPC= hit count=9976 mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.batch_stats= snapshot_time: 1789022982.938476100 secs.nsecs start_time: 1789022862.521621236 secs.nsecs elapsed_time: 120.416854864 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.batch_stats= snapshot_time: 1789022982.938543221 secs.nsecs start_time: 1789022862.521715867 secs.nsecs elapsed_time: 120.416827354 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.batch_stats=0 llite.lustre-ffff9c1c993d3000.statahead_max=2048 llite.lustre-ffff9c1c993d3000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 2.65 seconds: 37.69 ops/second llite.lustre-ffff9c1c993d3000.statahead_max=0 101 real 0m2.663s user 0m0.006s sys 0m0.301s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff9c1c993d3000.statahead_max=2048 llite.lustre-ffff9c1c993d3000.statahead_max=2048 101 real 0m1.190s user 0m0.005s sys 0m0.176s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10075 miss_total: 25 swrong=0 ewrong=0 - open/close 839 (time 1789023258.32 total 10.01 last 83.81) total: 900 open/close in 10.87 seconds: 82.81 ops/second llite.lustre-ffff9c1c993d3000.statahead_max=0 1001 real 0m19.647s user 0m0.033s sys 0m2.840s 'ls -l' 1000 files without statahead: 19 sec llite.lustre-ffff9c1c993d3000.statahead_max=2048 llite.lustre-ffff9c1c993d3000.statahead_max=2048 1001 real 0m7.558s user 0m0.026s sys 0m1.321s 'ls -l' 1000 files with statahead: 7 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11074 miss_total: 26 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 20 seconds llite.lustre-ffff9c1c993d3000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11074 miss_total: 26 mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.batch_stats= snapshot_time: 1789023319.154599614 secs.nsecs start_time: 1789022985.260596288 secs.nsecs elapsed_time: 333.894003326 secs.nsecs subreqs per batch batches %% cum %% 1: 1 14 14 2: 0 0 14 4: 3 42 57 8: 0 0 57 16: 0 0 57 32: 1 14 71 64: 0 0 71 128: 0 0 71 256: 2 28 100 mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.batch_stats= snapshot_time: 1789023319.154666975 secs.nsecs start_time: 1789022985.260743935 secs.nsecs elapsed_time: 333.893923040 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 10 10 8: 4 40 50 16: 0 0 50 32: 2 20 70 64: 1 10 80 128: 0 0 80 256: 2 20 100 - open/close 814 (time 1789023332.36 total 10.01 last 81.35) - open/close 1709 (time 1789023342.37 total 20.01 last 89.45) - open/close 2434 (time 1789023352.37 total 30.02 last 72.45) - open/close 3199 (time 1789023362.38 total 40.02 last 76.48) - open/close 4004 (time 1789023372.39 total 50.04 last 80.35) - open/close 4932 (time 1789023382.40 total 60.04 last 92.77) - open/close 5930 (time 1789023392.42 total 70.06 last 99.58) - open/close 6798 (time 1789023402.42 total 80.07 last 86.78) - open/close 7610 (time 1789023412.42 total 90.07 last 81.19) - open/close 8257 (time 1789023422.42 total 100.07 last 64.69) - open/close 9055 (time 1789023432.43 total 110.07 last 79.77) - open/close 9863 (time 1789023442.43 total 120.07 last 80.80) total: 10000 open/close in 121.61 seconds: 82.23 ops/second llite.lustre-ffff9c1c993d3000.statahead_batch_max=0 llite.lustre-ffff9c1c993d3000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.stats=clear mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m1.874s user 0m0.285s sys 0m13.788s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9c1c993d3000.statahead_batch_max=256 llite.lustre-ffff9c1c993d3000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.stats=clear mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m49.991s user 0m0.299s sys 0m14.161s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9c1c993d3000.batch_stats= snapshot_time: 1789023574.940050067 secs.nsecs start_time: 1789023514.410880407 secs.nsecs elapsed_time: 60.529169660 secs.nsecs subreqs per batch batches %% cum %% 1: 119 43 43 2: 42 15 58 4: 27 9 68 8: 19 6 75 16: 10 3 78 32: 18 6 85 64: 6 2 87 128: 1 0 87 256: 34 12 100 mdc.lustre-MDT0001-mdc-ffff9c1c993d3000.batch_stats= snapshot_time: 1789023574.940291799 secs.nsecs start_time: 1789023514.413437821 secs.nsecs elapsed_time: 60.526853978 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9c1c993d3000.statahead_batch_max=64 llite.lustre-ffff9c1c993d3000.statahead_max=128