== sanity test 123ad: Verify batching statahead works correctly ========================================================== 23:49:34 (1786765774) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff8eeec6410000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8eeec6410000.batch_stats=0 llite.lustre-ffff8eeec6410000.statahead_max=32 llite.lustre-ffff8eeec6410000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff8eeec6410000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8eeec6410000.batch_stats=0 striped dir -i1 -c2 -H all_char /mnt/lustre/d123ad.sanity total: 100 open/close in 2.76 seconds: 36.26 ops/second llite.lustre-ffff8eeec6410000.statahead_max=0 101 real 0m2.240s user 0m0.004s sys 0m0.348s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff8eeec6410000.statahead_max=32 llite.lustre-ffff8eeec6410000.statahead_max=32 101 real 0m0.973s user 0m0.004s sys 0m0.208s 'ls -l' 100 files with statahead: 1 sec statahead total: 111 statahead wrong: 2 agl total: 105 list_total: 31 fname_total: 75 hit_total: 3955 miss_total: 1573 swrong=2 ewrong=2 - open/close 799 (time 1786765801.49 total 10.00 last 79.87) total: 900 open/close in 11.30 seconds: 79.62 ops/second llite.lustre-ffff8eeec6410000.statahead_max=0 1001 real 0m22.463s user 0m0.034s sys 0m3.363s 'ls -l' 1000 files without statahead: 22 sec llite.lustre-ffff8eeec6410000.statahead_max=32 llite.lustre-ffff8eeec6410000.statahead_max=32 1001 real 0m7.854s user 0m0.027s sys 0m1.681s 'ls -l' 1000 files with statahead: 8 sec statahead total: 112 statahead wrong: 2 agl total: 106 list_total: 32 fname_total: 75 hit_total: 4952 miss_total: 1576 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 19 seconds llite.lustre-ffff8eeec6410000.statahead_stats= statahead total: 112 statahead wrong: 2 agl total: 106 list_total: 32 fname_total: 75 hit_total: 4952 miss_total: 1576 mdc.lustre-MDT0000-mdc-ffff8eeec6410000.batch_stats= snapshot_time: 1786765862.891293059 secs.nsecs start_time: 1786765779.471310189 secs.nsecs elapsed_time: 83.419982870 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 3 7 7 8: 2 5 12 16: 20 51 64 32: 14 35 100 mdc.lustre-MDT0001-mdc-ffff8eeec6410000.batch_stats= snapshot_time: 1786765862.891406681 secs.nsecs start_time: 1786765779.471850519 secs.nsecs elapsed_time: 83.419556162 secs.nsecs subreqs per batch batches %% cum %% 1: 2 5 5 2: 0 0 5 4: 2 5 10 8: 3 7 17 16: 18 46 64 32: 14 35 100 - open/close 701 (time 1786765875.47 total 10.00 last 70.08) - open/close 1388 (time 1786765885.49 total 20.02 last 68.60) - open/close 2073 (time 1786765895.50 total 30.03 last 68.44) - open/close 2782 (time 1786765905.50 total 40.03 last 70.87) - open/close 3445 (time 1786765915.51 total 50.04 last 66.25) - open/close 4161 (time 1786765925.51 total 60.04 last 71.60) - open/close 4878 (time 1786765935.51 total 70.04 last 71.67) - open/close 5636 (time 1786765945.52 total 80.05 last 75.76) - open/close 6383 (time 1786765955.53 total 90.06 last 74.64) - open/close 7139 (time 1786765965.53 total 100.06 last 75.57) - open/close 7834 (time 1786765975.54 total 110.07 last 69.44) - open/close 8445 (time 1786765985.54 total 120.07 last 61.10) - open/close 9185 (time 1786765995.55 total 130.08 last 73.94) - open/close 9897 (time 1786766005.57 total 140.10 last 71.04) total: 10000 open/close in 141.34 seconds: 70.75 ops/second llite.lustre-ffff8eeec6410000.statahead_batch_max=0 llite.lustre-ffff8eeec6410000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8eeec6410000.stats=clear mdc.lustre-MDT0001-mdc-ffff8eeec6410000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m59.194s user 0m0.315s sys 0m18.037s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8eeec6410000.statahead_batch_max=32 llite.lustre-ffff8eeec6410000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8eeec6410000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8eeec6410000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8eeec6410000.stats=clear mdc.lustre-MDT0001-mdc-ffff8eeec6410000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m48.609s user 0m0.307s sys 0m16.398s batched statahead RPC= hit count=9987 mdc.lustre-MDT0000-mdc-ffff8eeec6410000.batch_stats= snapshot_time: 1786766197.302284357 secs.nsecs start_time: 1786766075.384862584 secs.nsecs elapsed_time: 121.917421773 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff8eeec6410000.batch_stats= snapshot_time: 1786766197.302361157 secs.nsecs start_time: 1786766075.385002147 secs.nsecs elapsed_time: 121.917359010 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff8eeec6410000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8eeec6410000.batch_stats=0 llite.lustre-ffff8eeec6410000.statahead_max=2048 llite.lustre-ffff8eeec6410000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff8eeec6410000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8eeec6410000.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.40 seconds: 29.41 ops/second llite.lustre-ffff8eeec6410000.statahead_max=0 101 real 0m2.664s user 0m0.008s sys 0m0.362s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff8eeec6410000.statahead_max=2048 llite.lustre-ffff8eeec6410000.statahead_max=2048 101 real 0m1.160s user 0m0.002s sys 0m0.215s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10086 miss_total: 14 swrong=0 ewrong=0 - open/close 708 (time 1786766552.14 total 10.01 last 70.74) total: 900 open/close in 13.40 seconds: 67.15 ops/second llite.lustre-ffff8eeec6410000.statahead_max=0 1001 real 0m26.303s user 0m0.043s sys 0m3.609s 'ls -l' 1000 files without statahead: 26 sec llite.lustre-ffff8eeec6410000.statahead_max=2048 llite.lustre-ffff8eeec6410000.statahead_max=2048 1001 real 0m7.981s user 0m0.027s sys 0m1.607s 'ls -l' 1000 files with statahead: 8 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11085 miss_total: 15 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 23 seconds llite.lustre-ffff8eeec6410000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11085 miss_total: 15 mdc.lustre-MDT0000-mdc-ffff8eeec6410000.batch_stats= snapshot_time: 1786766628.059267515 secs.nsecs start_time: 1786766200.496303807 secs.nsecs elapsed_time: 427.562963708 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 14 14 8: 3 42 57 16: 0 0 57 32: 1 14 71 64: 0 0 71 128: 0 0 71 256: 2 28 100 mdc.lustre-MDT0001-mdc-ffff8eeec6410000.batch_stats= snapshot_time: 1786766628.059356745 secs.nsecs start_time: 1786766200.496516710 secs.nsecs elapsed_time: 427.562840035 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 18 18 8: 1 9 27 16: 4 36 63 32: 1 9 72 64: 1 9 81 128: 0 0 81 256: 2 18 100 - open/close 720 (time 1786766640.73 total 10.00 last 71.99) - open/close 1348 (time 1786766650.74 total 20.01 last 62.77) - open/close 2031 (time 1786766660.74 total 30.01 last 68.27) - open/close 2583 (time 1786766670.75 total 40.02 last 55.18) - open/close 3158 (time 1786766680.75 total 50.02 last 57.48) - open/close 3856 (time 1786766690.75 total 60.02 last 69.79) - open/close 4518 (time 1786766700.76 total 70.03 last 66.13) - open/close 5211 (time 1786766710.77 total 80.04 last 69.24) - open/close 5945 (time 1786766720.80 total 90.07 last 73.19) - open/close 6517 (time 1786766730.81 total 100.08 last 57.15) - open/close 7103 (time 1786766740.85 total 110.12 last 58.38) - open/close 7568 (time 1786766750.87 total 120.13 last 46.42) - open/close 8130 (time 1786766760.87 total 130.14 last 56.17) - open/close 8711 (time 1786766770.87 total 140.14 last 58.09) - open/close 9254 (time 1786766780.90 total 150.17 last 54.17) - open/close 9847 (time 1786766790.90 total 160.17 last 59.28) total: 10000 open/close in 161.64 seconds: 61.87 ops/second llite.lustre-ffff8eeec6410000.statahead_batch_max=0 llite.lustre-ffff8eeec6410000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8eeec6410000.stats=clear mdc.lustre-MDT0001-mdc-ffff8eeec6410000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m11.645s user 0m0.365s sys 0m17.908s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8eeec6410000.statahead_batch_max=256 llite.lustre-ffff8eeec6410000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8eeec6410000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8eeec6410000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8eeec6410000.stats=clear mdc.lustre-MDT0001-mdc-ffff8eeec6410000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m6.527s user 0m0.294s sys 0m16.978s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff8eeec6410000.batch_stats= snapshot_time: 1786766956.560214210 secs.nsecs start_time: 1786766873.157154954 secs.nsecs elapsed_time: 83.403059256 secs.nsecs subreqs per batch batches %% cum %% 1: 133 45 45 2: 41 13 58 4: 36 12 71 8: 18 6 77 16: 12 4 81 32: 10 3 84 64: 9 3 87 128: 2 0 88 256: 34 11 100 mdc.lustre-MDT0001-mdc-ffff8eeec6410000.batch_stats= snapshot_time: 1786766956.560281881 secs.nsecs start_time: 1786766873.157294327 secs.nsecs elapsed_time: 83.402987554 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff8eeec6410000.statahead_batch_max=64 llite.lustre-ffff8eeec6410000.statahead_max=128