== sanity test 123ad: Verify batching statahead works correctly ========================================================== 02:34:21 (1790663661) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9484c70db000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9484c70db000.batch_stats=0 llite.lustre-ffff9484c70db000.statahead_max=32 llite.lustre-ffff9484c70db000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9484c70db000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9484c70db000.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 2.91 seconds: 34.39 ops/second llite.lustre-ffff9484c70db000.statahead_max=0 101 real 0m2.512s user 0m0.007s sys 0m0.352s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff9484c70db000.statahead_max=32 llite.lustre-ffff9484c70db000.statahead_max=32 101 real 0m1.231s user 0m0.004s sys 0m0.190s 'ls -l' 100 files with statahead: 1 sec statahead total: 164 statahead wrong: 2 agl total: 158 list_total: 31 fname_total: 124 hit_total: 4448 miss_total: 1287 swrong=2 ewrong=2 - open/close 532 (time 1790663690.18 total 10.01 last 53.14) total: 900 open/close in 17.76 seconds: 50.69 ops/second llite.lustre-ffff9484c70db000.statahead_max=0 1001 real 0m23.662s user 0m0.027s sys 0m3.291s 'ls -l' 1000 files without statahead: 24 sec llite.lustre-ffff9484c70db000.statahead_max=32 llite.lustre-ffff9484c70db000.statahead_max=32 1001 real 0m11.407s user 0m0.038s sys 0m1.524s 'ls -l' 1000 files with statahead: 11 sec statahead total: 165 statahead wrong: 2 agl total: 159 list_total: 32 fname_total: 124 hit_total: 5445 miss_total: 1290 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 24 seconds llite.lustre-ffff9484c70db000.statahead_stats= statahead total: 165 statahead wrong: 2 agl total: 159 list_total: 32 fname_total: 124 hit_total: 5445 miss_total: 1290 mdc.lustre-MDT0000-mdc-ffff9484c70db000.batch_stats= snapshot_time: 1790663772.468588179 secs.nsecs start_time: 1790663666.204997362 secs.nsecs elapsed_time: 106.263590817 secs.nsecs subreqs per batch batches %% cum %% 1: 4 10 10 2: 2 5 15 4: 2 5 20 8: 1 2 23 16: 26 66 89 32: 4 10 100 mdc.lustre-MDT0001-mdc-ffff9484c70db000.batch_stats= snapshot_time: 1790663772.468687792 secs.nsecs start_time: 1790663666.205113987 secs.nsecs elapsed_time: 106.263573805 secs.nsecs subreqs per batch batches %% cum %% 1: 2 4 4 2: 1 2 7 4: 1 2 9 8: 3 7 17 16: 10 24 41 32: 24 58 100 - open/close 665 (time 1790663786.94 total 10.00 last 66.48) - open/close 1380 (time 1790663796.94 total 20.01 last 71.48) - open/close 2037 (time 1790663806.95 total 30.01 last 65.70) - open/close 2765 (time 1790663816.96 total 40.02 last 72.72) - open/close 3577 (time 1790663826.96 total 50.02 last 81.15) - open/close 4335 (time 1790663836.97 total 60.03 last 75.75) - open/close 4936 (time 1790663846.97 total 70.03 last 60.09) - open/close 5483 (time 1790663856.98 total 80.04 last 54.65) - open/close 5935 (time 1790663866.99 total 90.05 last 45.14) - open/close 6491 (time 1790663877.01 total 100.08 last 55.49) - open/close 7110 (time 1790663887.03 total 110.09 last 61.82) - open/close 7779 (time 1790663897.04 total 120.10 last 66.83) - open/close 8427 (time 1790663907.05 total 130.11 last 64.74) - open/close 9101 (time 1790663917.05 total 140.11 last 67.37) - open/close 9739 (time 1790663927.06 total 150.12 last 63.76) total: 10000 open/close in 154.98 seconds: 64.52 ops/second llite.lustre-ffff9484c70db000.statahead_batch_max=0 llite.lustre-ffff9484c70db000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9484c70db000.stats=clear mdc.lustre-MDT0001-mdc-ffff9484c70db000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m3.984s user 0m0.381s sys 0m15.679s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9484c70db000.statahead_batch_max=32 llite.lustre-ffff9484c70db000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9484c70db000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9484c70db000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9484c70db000.stats=clear mdc.lustre-MDT0001-mdc-ffff9484c70db000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m39.712s user 0m0.332s sys 0m14.345s batched statahead RPC= hit count=9979 mdc.lustre-MDT0000-mdc-ffff9484c70db000.batch_stats= snapshot_time: 1790664116.589847027 secs.nsecs start_time: 1790664006.243927497 secs.nsecs elapsed_time: 110.345919530 secs.nsecs subreqs per batch batches %% cum %% 1: 1 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9484c70db000.batch_stats= snapshot_time: 1790664116.589928701 secs.nsecs start_time: 1790664006.244070677 secs.nsecs elapsed_time: 110.345858024 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9484c70db000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9484c70db000.batch_stats=0 llite.lustre-ffff9484c70db000.statahead_max=2048 llite.lustre-ffff9484c70db000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9484c70db000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9484c70db000.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.24 seconds: 30.90 ops/second llite.lustre-ffff9484c70db000.statahead_max=0 101 real 0m3.256s user 0m0.006s sys 0m0.387s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff9484c70db000.statahead_max=2048 llite.lustre-ffff9484c70db000.statahead_max=2048 101 real 0m0.940s user 0m0.006s sys 0m0.180s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10078 miss_total: 22 swrong=0 ewrong=0 - open/close 740 (time 1790664458.82 total 10.01 last 73.91) total: 900 open/close in 12.37 seconds: 72.78 ops/second llite.lustre-ffff9484c70db000.statahead_max=0 1001 real 0m22.179s user 0m0.042s sys 0m3.375s 'ls -l' 1000 files without statahead: 23 sec llite.lustre-ffff9484c70db000.statahead_max=2048 llite.lustre-ffff9484c70db000.statahead_max=2048 1001 real 0m7.664s user 0m0.037s sys 0m1.591s 'ls -l' 1000 files with statahead: 8 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11077 miss_total: 23 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 27 seconds llite.lustre-ffff9484c70db000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11077 miss_total: 23 mdc.lustre-MDT0000-mdc-ffff9484c70db000.batch_stats= snapshot_time: 1790664531.715134076 secs.nsecs start_time: 1790664118.741698040 secs.nsecs elapsed_time: 412.973436036 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 10 10 8: 5 50 60 16: 1 10 70 32: 1 10 80 64: 0 0 80 128: 0 0 80 256: 2 20 100 mdc.lustre-MDT0001-mdc-ffff9484c70db000.batch_stats= snapshot_time: 1790664531.715534347 secs.nsecs start_time: 1790664118.741845228 secs.nsecs elapsed_time: 412.973689119 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 2 20 20 4: 1 10 30 8: 1 10 40 16: 3 30 70 32: 1 10 80 64: 0 0 80 128: 0 0 80 256: 2 20 100 - open/close 610 (time 1790664544.84 total 10.02 last 60.89) - open/close 1289 (time 1790664554.84 total 20.02 last 67.87) - open/close 1975 (time 1790664564.86 total 30.05 last 68.44) - open/close 2600 (time 1790664574.87 total 40.05 last 62.46) - open/close 3239 (time 1790664584.88 total 50.06 last 63.85) - open/close 3967 (time 1790664594.89 total 60.07 last 72.74) - open/close 4651 (time 1790664604.90 total 70.08 last 68.31) - open/close 5225 (time 1790664614.91 total 80.09 last 57.35) - open/close 5927 (time 1790664624.94 total 90.12 last 70.00) - open/close 6573 (time 1790664634.96 total 100.14 last 64.43) - open/close 7133 (time 1790664644.97 total 110.15 last 55.97) - open/close 7677 (time 1790664654.97 total 120.15 last 54.39) - open/close 8150 (time 1790664664.97 total 130.16 last 47.28) - open/close 8755 (time 1790664674.98 total 140.16 last 60.45) - open/close 9357 (time 1790664684.99 total 150.17 last 60.18) total: 10000 open/close in 159.81 seconds: 62.57 ops/second llite.lustre-ffff9484c70db000.statahead_batch_max=0 llite.lustre-ffff9484c70db000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9484c70db000.stats=clear mdc.lustre-MDT0001-mdc-ffff9484c70db000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m3.424s user 0m0.355s sys 0m15.982s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9484c70db000.statahead_batch_max=256 llite.lustre-ffff9484c70db000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9484c70db000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9484c70db000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9484c70db000.stats=clear mdc.lustre-MDT0001-mdc-ffff9484c70db000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m59.372s user 0m0.339s sys 0m15.105s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9484c70db000.batch_stats= snapshot_time: 1790664841.489872365 secs.nsecs start_time: 1790664768.814981198 secs.nsecs elapsed_time: 72.674891167 secs.nsecs subreqs per batch batches %% cum %% 1: 88 40 40 2: 28 12 52 4: 20 9 61 8: 13 5 67 16: 10 4 72 32: 20 9 81 64: 5 2 83 128: 2 0 84 256: 34 15 100 mdc.lustre-MDT0001-mdc-ffff9484c70db000.batch_stats= snapshot_time: 1790664841.489943275 secs.nsecs start_time: 1790664768.815063118 secs.nsecs elapsed_time: 72.674880157 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9484c70db000.statahead_batch_max=64 llite.lustre-ffff9484c70db000.statahead_max=128