== sanity test 123ad: Verify batching statahead works correctly ========================================================== 21:39:58 (1745458798) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats=0 mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats=0 llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=32 llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats=0 mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 0.60 seconds: 166.36 ops/second llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=0 101 real 0m0.537s user 0m0.001s sys 0m0.197s ls -l 100 files without statahead: 1 sec llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=32 32 101 real 0m0.342s user 0m0.002s sys 0m0.130s ls -l 100 files with statahead: 0 sec statahead total: 44 statahead wrong: 1 agl total: 44 list_total: 19 fname_total: 25 hit_total: 11339 miss_total: 218 total: 900 open/close in 1.90 seconds: 473.63 ops/second llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=0 1001 real 0m7.100s user 0m0.036s sys 0m2.679s ls -l 1000 files without statahead: 7 sec llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=32 32 1001 real 0m1.690s user 0m0.017s sys 0m0.868s ls -l 1000 files with statahead: 2 sec statahead total: 45 statahead wrong: 1 agl total: 45 list_total: 20 fname_total: 25 hit_total: 12338 miss_total: 219 - open/close 5016 (time 1745458825.53 total 10.00 last 501.59) total: 9000 open/close in 18.50 seconds: 486.58 ops/second llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=0 10001 real 0m51.444s user 0m0.280s sys 0m18.644s ls -l 10000 files without statahead: 51 sec llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=32 32 10001 real 0m16.939s user 0m0.218s sys 0m8.954s ls -l 10000 files with statahead: 17 sec statahead total: 46 statahead wrong: 1 agl total: 46 list_total: 21 fname_total: 25 hit_total: 22337 miss_total: 220 ls -l done rm -r /mnt/lustre/d123ad.sanity: 36 seconds llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_stats= statahead total: 46 statahead wrong: 1 agl total: 46 list_total: 21 fname_total: 25 hit_total: 22337 miss_total: 220 mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats= snapshot_time: 1745458942.858752825 secs.nsecs start_time: 1745458799.788483564 secs.nsecs elapsed_time: 143.070269261 secs.nsecs subreqs per batch batches % cum % 1: 1 0 0 2: 1 0 0 4: 3 0 1 8: 6 1 3 16: 199 56 59 32: 145 40 100 mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats= snapshot_time: 1745458942.858846833 secs.nsecs start_time: 1745458799.788546954 secs.nsecs elapsed_time: 143.070299879 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 2: 1 0 0 4: 5 1 1 8: 5 1 3 16: 196 55 58 32: 147 41 100 - open/close 4549 (time 1745458953.45 total 10.00 last 454.89) - open/close 8912 (time 1745458963.45 total 20.00 last 436.24) total: 10000 open/close in 21.97 seconds: 455.23 ops/second llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_batch_max=0 llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_stats=clear mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.stats=clear mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.stats=clear ls -l files in d123ad.sanity: 10001 real 0m10.375s user 0m0.199s sys 0m9.698s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_batch_max=32 llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_stats=clear mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats=clear mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats=clear mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.stats=clear mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.stats=clear ls -l files in d123ad.sanity: 10001 real 0m20.653s user 0m0.190s sys 0m9.157s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats= snapshot_time: 1745459000.586423493 secs.nsecs start_time: 1745458977.657971375 secs.nsecs elapsed_time: 22.928452118 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 2 0 0 32: 312 99 100 mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats= snapshot_time: 1745459000.586527298 secs.nsecs start_time: 1745458977.658016590 secs.nsecs elapsed_time: 22.928510708 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats=0 mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats=0 llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=2048 llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats=0 mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 0.65 seconds: 152.73 ops/second llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=0 101 real 0m0.532s user 0m0.002s sys 0m0.202s ls -l 100 files without statahead: 0 sec llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=2048 2048 101 real 0m0.315s user 0m0.004s sys 0m0.125s ls -l 100 files with statahead: 0 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10098 miss_total: 2 total: 900 open/close in 1.83 seconds: 492.08 ops/second llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=0 1001 real 0m5.931s user 0m0.030s sys 0m2.062s ls -l 1000 files without statahead: 6 sec llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=2048 2048 1001 real 0m1.273s user 0m0.011s sys 0m0.862s ls -l 1000 files with statahead: 1 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11097 miss_total: 3 - open/close 4280 (time 1745459088.76 total 10.00 last 427.99) total: 9000 open/close in 19.98 seconds: 450.44 ops/second llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=0 10001 real 0m53.476s user 0m0.240s sys 0m19.206s ls -l 10000 files without statahead: 54 sec llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=2048 2048 10001 real 0m9.871s user 0m0.207s sys 0m8.503s ls -l 10000 files with statahead: 10 sec statahead total: 4 statahead wrong: 0 agl total: 4 list_total: 4 fname_total: 0 hit_total: 21096 miss_total: 4 ls -l done rm -r /mnt/lustre/d123ad.sanity: 41 seconds llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_stats= statahead total: 4 statahead wrong: 0 agl total: 4 list_total: 4 fname_total: 0 hit_total: 21096 miss_total: 4 mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats= snapshot_time: 1745459208.285012941 secs.nsecs start_time: 1745459001.069219767 secs.nsecs elapsed_time: 207.215793174 secs.nsecs subreqs per batch batches % cum % 1: 7 14 14 2: 2 4 18 4: 2 4 22 8: 2 4 26 16: 1 2 28 32: 0 0 28 64: 1 2 30 128: 7 14 44 256: 27 55 100 mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats= snapshot_time: 1745459208.285079936 secs.nsecs start_time: 1745459001.069296184 secs.nsecs elapsed_time: 207.215783752 secs.nsecs subreqs per batch batches % cum % 1: 4 7 7 2: 1 1 9 4: 1 1 11 8: 4 7 19 16: 1 1 21 32: 3 5 27 64: 2 3 31 128: 2 3 35 256: 33 64 100 - open/close 4902 (time 1745459218.97 total 10.00 last 490.16) - open/close 7849 (time 1745459228.98 total 20.00 last 294.67) total: 10000 open/close in 27.35 seconds: 365.59 ops/second llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_batch_max=0 llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_stats=clear mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.stats=clear mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.stats=clear ls -l files in d123ad.sanity: 10001 real 0m10.212s user 0m0.173s sys 0m9.125s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_batch_max=256 llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_stats=clear mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats=clear mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats=clear mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.stats=clear mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.stats=clear ls -l files in d123ad.sanity: 10001 real 0m10.051s user 0m0.197s sys 0m8.733s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats= snapshot_time: 1745459260.770113382 secs.nsecs start_time: 1745459248.360545094 secs.nsecs elapsed_time: 12.409568288 secs.nsecs subreqs per batch batches % cum % 1: 267 68 68 2: 12 3 71 4: 12 3 74 8: 20 5 79 16: 22 5 85 32: 20 5 90 64: 1 0 91 128: 1 0 91 256: 34 8 100 mdc.lustre-MDT0001-mdc-ca81d161-d577-43c5-b98d-b65a09a0e0ef.batch_stats= snapshot_time: 1745459260.770177032 secs.nsecs start_time: 1745459248.360609172 secs.nsecs elapsed_time: 12.409567860 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_batch_max=64 llite.lustre-ca81d161-d577-43c5-b98d-b65a09a0e0ef.statahead_max=128