== sanity test 123ad: Verify batching statahead works correctly ========================================================== 22:32:09 (1787797929) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.batch_stats=0 llite.lustre-ffff8e1ed1202800.statahead_max=32 llite.lustre-ffff8e1ed1202800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 1.98 seconds: 50.62 ops/second llite.lustre-ffff8e1ed1202800.statahead_max=0 101 real 0m2.661s user 0m0.003s sys 0m0.294s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff8e1ed1202800.statahead_max=32 llite.lustre-ffff8e1ed1202800.statahead_max=32 101 real 0m1.345s user 0m0.002s sys 0m0.186s 'ls -l' 100 files with statahead: 2 sec statahead total: 135 statahead wrong: 2 agl total: 130 list_total: 35 fname_total: 100 hit_total: 4081 miss_total: 1382 swrong=2 ewrong=2 - open/close 840 (time 1787797958.61 total 10.00 last 83.97) total: 900 open/close in 10.78 seconds: 83.51 ops/second llite.lustre-ffff8e1ed1202800.statahead_max=0 1001 real 0m19.999s user 0m0.031s sys 0m2.746s 'ls -l' 1000 files without statahead: 20 sec llite.lustre-ffff8e1ed1202800.statahead_max=32 llite.lustre-ffff8e1ed1202800.statahead_max=32 1001 real 0m8.583s user 0m0.025s sys 0m1.352s 'ls -l' 1000 files with statahead: 8 sec statahead total: 136 statahead wrong: 2 agl total: 131 list_total: 36 fname_total: 100 hit_total: 5078 miss_total: 1385 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 18 seconds llite.lustre-ffff8e1ed1202800.statahead_stats= statahead total: 136 statahead wrong: 2 agl total: 131 list_total: 36 fname_total: 100 hit_total: 5078 miss_total: 1385 mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.batch_stats= snapshot_time: 1787798019.251101033 secs.nsecs start_time: 1787797934.371519456 secs.nsecs elapsed_time: 84.879581577 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 2 5 5 4: 3 8 13 8: 1 2 16 16: 26 72 88 32: 4 11 100 mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.batch_stats= snapshot_time: 1787798019.251339030 secs.nsecs start_time: 1787797934.371615992 secs.nsecs elapsed_time: 84.879723038 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 2 2 8: 3 7 10 16: 13 33 43 32: 22 56 100 - open/close 937 (time 1787798032.10 total 10.00 last 93.68) - open/close 1826 (time 1787798042.10 total 20.00 last 88.89) - open/close 2748 (time 1787798052.11 total 30.01 last 92.13) - open/close 3668 (time 1787798062.13 total 40.02 last 91.87) - open/close 4591 (time 1787798072.14 total 50.04 last 92.20) - open/close 5478 (time 1787798082.14 total 60.04 last 88.66) - open/close 6464 (time 1787798092.15 total 70.04 last 98.55) - open/close 7446 (time 1787798102.16 total 80.06 last 98.10) - open/close 8366 (time 1787798112.16 total 90.06 last 91.93) - open/close 9291 (time 1787798122.16 total 100.06 last 92.50) total: 10000 open/close in 108.58 seconds: 92.10 ops/second llite.lustre-ffff8e1ed1202800.statahead_batch_max=0 llite.lustre-ffff8e1ed1202800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.stats=clear mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m57.434s user 0m0.299s sys 0m15.093s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8e1ed1202800.statahead_batch_max=32 llite.lustre-ffff8e1ed1202800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.stats=clear mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m34.339s user 0m0.259s sys 0m13.918s batched statahead RPC= hit count=9980 mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.batch_stats= snapshot_time: 1787798300.336656888 secs.nsecs start_time: 1787798196.882700096 secs.nsecs elapsed_time: 103.453956792 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.batch_stats= snapshot_time: 1787798300.336726478 secs.nsecs start_time: 1787798196.882763656 secs.nsecs elapsed_time: 103.453962822 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.batch_stats=0 llite.lustre-ffff8e1ed1202800.statahead_max=2048 llite.lustre-ffff8e1ed1202800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.10 seconds: 47.59 ops/second llite.lustre-ffff8e1ed1202800.statahead_max=0 101 real 0m2.246s user 0m0.006s sys 0m0.298s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff8e1ed1202800.statahead_max=2048 llite.lustre-ffff8e1ed1202800.statahead_max=2048 101 real 0m0.860s user 0m0.004s sys 0m0.193s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10079 miss_total: 21 swrong=0 ewrong=0 total: 900 open/close in 9.70 seconds: 92.76 ops/second llite.lustre-ffff8e1ed1202800.statahead_max=0 1001 real 0m19.893s user 0m0.030s sys 0m2.842s 'ls -l' 1000 files without statahead: 20 sec llite.lustre-ffff8e1ed1202800.statahead_max=2048 llite.lustre-ffff8e1ed1202800.statahead_max=2048 1001 real 0m6.178s user 0m0.026s sys 0m1.327s 'ls -l' 1000 files with statahead: 6 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11078 miss_total: 22 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 17 seconds llite.lustre-ffff8e1ed1202800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11078 miss_total: 22 mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.batch_stats= snapshot_time: 1787798628.994769716 secs.nsecs start_time: 1787798302.995893351 secs.nsecs elapsed_time: 325.998876365 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 33 33 8: 2 33 66 16: 0 0 66 32: 0 0 66 64: 0 0 66 128: 0 0 66 256: 2 33 100 mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.batch_stats= snapshot_time: 1787798628.994855238 secs.nsecs start_time: 1787798302.996057822 secs.nsecs elapsed_time: 325.998797416 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 11 11 8: 2 22 33 16: 2 22 55 32: 1 11 66 64: 0 0 66 128: 1 11 77 256: 2 22 100 - open/close 829 (time 1787798641.58 total 10.01 last 82.86) - open/close 1643 (time 1787798651.59 total 20.01 last 81.37) - open/close 2611 (time 1787798661.59 total 30.01 last 96.74) - open/close 3387 (time 1787798671.60 total 40.02 last 77.54) - open/close 4185 (time 1787798681.60 total 50.02 last 79.80) - open/close 5117 (time 1787798691.60 total 60.03 last 93.17) - open/close 6008 (time 1787798701.61 total 70.03 last 89.05) - open/close 6809 (time 1787798711.61 total 80.03 last 80.10) - open/close 7558 (time 1787798721.62 total 90.04 last 74.85) - open/close 8339 (time 1787798731.62 total 100.04 last 78.08) - open/close 9140 (time 1787798741.62 total 110.04 last 80.09) - open/close 9984 (time 1787798751.64 total 120.06 last 84.26) total: 10000 open/close in 120.26 seconds: 83.15 ops/second llite.lustre-ffff8e1ed1202800.statahead_batch_max=0 llite.lustre-ffff8e1ed1202800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.stats=clear mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m59.069s user 0m0.341s sys 0m15.983s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8e1ed1202800.statahead_batch_max=256 llite.lustre-ffff8e1ed1202800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.stats=clear mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m49.867s user 0m0.323s sys 0m16.243s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff8e1ed1202800.batch_stats= snapshot_time: 1787798879.272538361 secs.nsecs start_time: 1787798818.251126073 secs.nsecs elapsed_time: 61.021412288 secs.nsecs subreqs per batch batches %% cum %% 1: 148 48 48 2: 37 12 61 4: 37 12 73 8: 15 4 78 16: 6 1 80 32: 19 6 86 64: 6 1 88 128: 1 0 88 256: 34 11 100 mdc.lustre-MDT0001-mdc-ffff8e1ed1202800.batch_stats= snapshot_time: 1787798879.272607978 secs.nsecs start_time: 1787798818.251266719 secs.nsecs elapsed_time: 61.021341259 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff8e1ed1202800.statahead_batch_max=64 llite.lustre-ffff8e1ed1202800.statahead_max=128