== sanity test 123ad: Verify batching statahead works correctly ========================================================== 23:04:55 (1788491095) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff90522424e800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff90522424e800.batch_stats=0 llite.lustre-ffff90522424e800.statahead_max=32 llite.lustre-ffff90522424e800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff90522424e800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff90522424e800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.17 seconds: 31.50 ops/second llite.lustre-ffff90522424e800.statahead_max=0 101 real 0m2.721s user 0m0.002s sys 0m0.338s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff90522424e800.statahead_max=32 llite.lustre-ffff90522424e800.statahead_max=32 101 real 0m1.180s user 0m0.006s sys 0m0.164s 'ls -l' 100 files with statahead: 1 sec statahead total: 128 statahead wrong: 2 agl total: 122 list_total: 30 fname_total: 76 hit_total: 3582 miss_total: 1652 swrong=2 ewrong=2 - open/close 637 (time 1788491125.50 total 10.01 last 63.67) total: 900 open/close in 13.85 seconds: 64.97 ops/second llite.lustre-ffff90522424e800.statahead_max=0 1001 real 0m23.827s user 0m0.031s sys 0m3.007s 'ls -l' 1000 files without statahead: 24 sec llite.lustre-ffff90522424e800.statahead_max=32 llite.lustre-ffff90522424e800.statahead_max=32 1001 real 0m9.528s user 0m0.037s sys 0m1.619s 'ls -l' 1000 files with statahead: 9 sec statahead total: 129 statahead wrong: 2 agl total: 123 list_total: 31 fname_total: 76 hit_total: 4581 miss_total: 1653 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 22 seconds llite.lustre-ffff90522424e800.statahead_stats= statahead total: 129 statahead wrong: 2 agl total: 123 list_total: 31 fname_total: 76 hit_total: 4581 miss_total: 1653 mdc.lustre-MDT0000-mdc-ffff90522424e800.batch_stats= snapshot_time: 1788491198.746660425 secs.nsecs start_time: 1788491100.540651366 secs.nsecs elapsed_time: 98.206009059 secs.nsecs subreqs per batch batches %% cum %% 1: 4 10 10 2: 1 2 13 4: 2 5 18 8: 1 2 21 16: 25 65 86 32: 5 13 100 mdc.lustre-MDT0001-mdc-ffff90522424e800.batch_stats= snapshot_time: 1788491198.746733901 secs.nsecs start_time: 1788491100.545165694 secs.nsecs elapsed_time: 98.201568207 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 1 2 5 8: 4 10 15 16: 12 30 45 32: 22 55 100 - open/close 704 (time 1788491211.44 total 10.01 last 70.33) - open/close 1403 (time 1788491221.46 total 20.02 last 69.80) - open/close 2121 (time 1788491231.46 total 30.03 last 71.75) - open/close 2819 (time 1788491241.47 total 40.03 last 69.78) - open/close 3516 (time 1788491251.47 total 50.04 last 69.69) - open/close 4108 (time 1788491261.48 total 60.04 last 59.15) - open/close 4616 (time 1788491271.49 total 70.06 last 50.74) - open/close 5270 (time 1788491281.49 total 80.06 last 65.39) - open/close 5974 (time 1788491291.50 total 90.07 last 70.35) - open/close 6560 (time 1788491301.50 total 100.07 last 58.60) - open/close 7064 (time 1788491311.50 total 110.07 last 50.38) - open/close 7611 (time 1788491321.51 total 120.08 last 54.65) - open/close 8245 (time 1788491331.52 total 130.09 last 63.33) - open/close 8943 (time 1788491341.53 total 140.10 last 69.74) - open/close 9625 (time 1788491351.53 total 150.10 last 68.20) total: 10000 open/close in 155.62 seconds: 64.26 ops/second llite.lustre-ffff90522424e800.statahead_batch_max=0 llite.lustre-ffff90522424e800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff90522424e800.stats=clear mdc.lustre-MDT0001-mdc-ffff90522424e800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m57.041s user 0m0.287s sys 0m15.523s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff90522424e800.statahead_batch_max=32 llite.lustre-ffff90522424e800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff90522424e800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff90522424e800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff90522424e800.stats=clear mdc.lustre-MDT0001-mdc-ffff90522424e800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m57.437s user 0m0.330s sys 0m14.592s batched statahead RPC= hit count=9978 mdc.lustre-MDT0000-mdc-ffff90522424e800.batch_stats= snapshot_time: 1788491550.900926214 secs.nsecs start_time: 1788491423.330265181 secs.nsecs elapsed_time: 127.570661033 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 1 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff90522424e800.batch_stats= snapshot_time: 1788491550.900992723 secs.nsecs start_time: 1788491423.330406994 secs.nsecs elapsed_time: 127.570585729 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff90522424e800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff90522424e800.batch_stats=0 llite.lustre-ffff90522424e800.statahead_max=2048 llite.lustre-ffff90522424e800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff90522424e800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff90522424e800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.89 seconds: 34.60 ops/second llite.lustre-ffff90522424e800.statahead_max=0 101 real 0m2.486s user 0m0.007s sys 0m0.330s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff90522424e800.statahead_max=2048 llite.lustre-ffff90522424e800.statahead_max=2048 101 real 0m1.251s user 0m0.006s sys 0m0.179s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10077 miss_total: 23 swrong=0 ewrong=0 - open/close 681 (time 1788491915.65 total 10.02 last 67.94) total: 900 open/close in 13.55 seconds: 66.44 ops/second llite.lustre-ffff90522424e800.statahead_max=0 1001 real 0m25.180s user 0m0.039s sys 0m3.032s 'ls -l' 1000 files without statahead: 25 sec llite.lustre-ffff90522424e800.statahead_max=2048 llite.lustre-ffff90522424e800.statahead_max=2048 1001 real 0m6.261s user 0m0.026s sys 0m1.356s 'ls -l' 1000 files with statahead: 7 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11076 miss_total: 24 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 25 seconds llite.lustre-ffff90522424e800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11076 miss_total: 24 mdc.lustre-MDT0000-mdc-ffff90522424e800.batch_stats= snapshot_time: 1788491988.942370784 secs.nsecs start_time: 1788491554.409599412 secs.nsecs elapsed_time: 434.532771372 secs.nsecs subreqs per batch batches %% cum %% 1: 1 14 14 2: 0 0 14 4: 3 42 57 8: 1 14 71 16: 0 0 71 32: 0 0 71 64: 0 0 71 128: 0 0 71 256: 2 28 100 mdc.lustre-MDT0001-mdc-ffff90522424e800.batch_stats= snapshot_time: 1788491988.942513227 secs.nsecs start_time: 1788491554.410977130 secs.nsecs elapsed_time: 434.531536097 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 4 44 44 16: 1 11 55 32: 1 11 66 64: 0 0 66 128: 1 11 77 256: 2 22 100 - open/close 472 (time 1788492002.59 total 10.01 last 47.14) - open/close 1075 (time 1788492012.59 total 20.02 last 60.29) - open/close 1699 (time 1788492022.60 total 30.02 last 62.35) - open/close 2354 (time 1788492032.61 total 40.04 last 65.41) - open/close 3015 (time 1788492042.62 total 50.05 last 66.04) - open/close 3719 (time 1788492052.62 total 60.05 last 70.38) - open/close 4437 (time 1788492062.64 total 70.07 last 71.68) - open/close 5156 (time 1788492072.65 total 80.07 last 71.86) - open/close 5933 (time 1788492082.66 total 90.08 last 77.62) - open/close 6626 (time 1788492092.66 total 100.08 last 69.28) - open/close 7295 (time 1788492102.66 total 110.09 last 66.89) - open/close 7930 (time 1788492112.66 total 120.09 last 63.50) - open/close 8562 (time 1788492122.67 total 130.10 last 63.12) - open/close 9295 (time 1788492132.68 total 140.10 last 73.27) - open/close 9968 (time 1788492142.70 total 150.13 last 67.12) total: 10000 open/close in 150.61 seconds: 66.40 ops/second llite.lustre-ffff90522424e800.statahead_batch_max=0 llite.lustre-ffff90522424e800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff90522424e800.stats=clear mdc.lustre-MDT0001-mdc-ffff90522424e800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m5.829s user 0m0.320s sys 0m14.956s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff90522424e800.statahead_batch_max=256 llite.lustre-ffff90522424e800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff90522424e800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff90522424e800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff90522424e800.stats=clear mdc.lustre-MDT0001-mdc-ffff90522424e800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m57.078s user 0m0.304s sys 0m14.786s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff90522424e800.batch_stats= snapshot_time: 1788492287.909792630 secs.nsecs start_time: 1788492218.710354938 secs.nsecs elapsed_time: 69.199437692 secs.nsecs subreqs per batch batches %% cum %% 1: 182 53 53 2: 45 13 66 4: 24 7 73 8: 19 5 78 16: 13 3 82 32: 16 4 87 64: 8 2 89 128: 1 0 90 256: 34 9 100 mdc.lustre-MDT0001-mdc-ffff90522424e800.batch_stats= snapshot_time: 1788492287.910074458 secs.nsecs start_time: 1788492218.710510308 secs.nsecs elapsed_time: 69.199564150 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff90522424e800.statahead_batch_max=64 llite.lustre-ffff90522424e800.statahead_max=128