== sanity test 123ad: Verify batching statahead works correctly ========================================================== 03:31:12 (1778657472) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9581988df800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9581988df800.batch_stats=0 llite.lustre-ffff9581988df800.statahead_max=32 llite.lustre-ffff9581988df800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9581988df800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9581988df800.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 2.65 seconds: 37.69 ops/second llite.lustre-ffff9581988df800.statahead_max=0 101 real 0m2.758s user 0m0.005s sys 0m0.345s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff9581988df800.statahead_max=32 llite.lustre-ffff9581988df800.statahead_max=32 101 real 0m1.185s user 0m0.003s sys 0m0.154s 'ls -l' 100 files with statahead: 1 sec statahead total: 144 statahead wrong: 2 agl total: 137 list_total: 38 fname_total: 97 hit_total: 4433 miss_total: 760 swrong=2 ewrong=2 - open/close 772 (time 1778657502.74 total 10.00 last 77.18) total: 900 open/close in 11.87 seconds: 75.84 ops/second llite.lustre-ffff9581988df800.statahead_max=0 1001 real 0m25.286s user 0m0.030s sys 0m3.161s 'ls -l' 1000 files without statahead: 25 sec llite.lustre-ffff9581988df800.statahead_max=32 llite.lustre-ffff9581988df800.statahead_max=32 1001 real 0m9.395s user 0m0.028s sys 0m1.519s 'ls -l' 1000 files with statahead: 9 sec statahead total: 145 statahead wrong: 2 agl total: 138 list_total: 39 fname_total: 97 hit_total: 5431 miss_total: 762 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 22 seconds llite.lustre-ffff9581988df800.statahead_stats= statahead total: 145 statahead wrong: 2 agl total: 138 list_total: 39 fname_total: 97 hit_total: 5431 miss_total: 762 mdc.lustre-MDT0000-mdc-ffff9581988df800.batch_stats= snapshot_time: 1778657574.493440452 secs.nsecs start_time: 1778657479.039306723 secs.nsecs elapsed_time: 95.454133729 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 2: 2 5 5 4: 2 5 11 8: 4 11 22 16: 19 52 75 32: 9 25 100 mdc.lustre-MDT0001-mdc-ffff9581988df800.batch_stats= snapshot_time: 1778657574.493611864 secs.nsecs start_time: 1778657479.039441257 secs.nsecs elapsed_time: 95.454170607 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 2: 1 2 2 4: 1 2 5 8: 2 5 10 16: 15 38 48 32: 20 51 100 - open/close 637 (time 1778657587.11 total 10.02 last 63.60) - open/close 1373 (time 1778657597.13 total 20.04 last 73.42) - open/close 2067 (time 1778657607.16 total 30.06 last 69.23) - open/close 2719 (time 1778657617.16 total 40.07 last 65.17) - open/close 3321 (time 1778657627.17 total 50.08 last 60.16) - open/close 4066 (time 1778657637.17 total 60.08 last 74.48) - open/close 4770 (time 1778657647.18 total 70.09 last 70.33) - open/close 5485 (time 1778657657.18 total 80.09 last 71.50) - open/close 6182 (time 1778657667.18 total 90.09 last 69.69) - open/close 6849 (time 1778657677.18 total 100.09 last 66.70) - open/close 7553 (time 1778657687.19 total 110.09 last 70.37) - open/close 8303 (time 1778657697.20 total 120.10 last 74.93) - open/close 9032 (time 1778657707.21 total 130.12 last 72.80) - open/close 9766 (time 1778657717.23 total 140.13 last 73.28) total: 10000 open/close in 143.85 seconds: 69.52 ops/second llite.lustre-ffff9581988df800.statahead_batch_max=0 llite.lustre-ffff9581988df800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9581988df800.stats=clear mdc.lustre-MDT0001-mdc-ffff9581988df800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m6.590s user 0m0.296s sys 0m16.109s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9581988df800.statahead_batch_max=32 llite.lustre-ffff9581988df800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9581988df800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9581988df800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9581988df800.stats=clear mdc.lustre-MDT0001-mdc-ffff9581988df800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m53.052s user 0m0.297s sys 0m15.088s batched statahead RPC= hit count=9977 mdc.lustre-MDT0000-mdc-ffff9581988df800.batch_stats= snapshot_time: 1778657928.292485251 secs.nsecs start_time: 1778657798.823423263 secs.nsecs elapsed_time: 129.469061988 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 2: 0 0 0 4: 1 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9581988df800.batch_stats= snapshot_time: 1778657928.292569681 secs.nsecs start_time: 1778657798.823512051 secs.nsecs elapsed_time: 129.469057630 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9581988df800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9581988df800.batch_stats=0 llite.lustre-ffff9581988df800.statahead_max=2048 llite.lustre-ffff9581988df800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9581988df800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9581988df800.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 2.58 seconds: 38.73 ops/second llite.lustre-ffff9581988df800.statahead_max=0 101 real 0m2.310s user 0m0.005s sys 0m0.338s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9581988df800.statahead_max=2048 llite.lustre-ffff9581988df800.statahead_max=2048 101 real 0m1.036s user 0m0.006s sys 0m0.222s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10076 miss_total: 24 swrong=0 ewrong=0 - open/close 756 (time 1778658261.79 total 10.01 last 75.55) total: 900 open/close in 11.99 seconds: 75.05 ops/second llite.lustre-ffff9581988df800.statahead_max=0 1001 real 0m23.855s user 0m0.031s sys 0m3.106s 'ls -l' 1000 files without statahead: 24 sec llite.lustre-ffff9581988df800.statahead_max=2048 llite.lustre-ffff9581988df800.statahead_max=2048 1001 real 0m7.140s user 0m0.028s sys 0m1.546s 'ls -l' 1000 files with statahead: 7 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11075 miss_total: 25 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 20 seconds llite.lustre-ffff9581988df800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11075 miss_total: 25 mdc.lustre-MDT0000-mdc-ffff9581988df800.batch_stats= snapshot_time: 1778658328.147826123 secs.nsecs start_time: 1778657930.880984886 secs.nsecs elapsed_time: 397.266841237 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 2: 1 16 16 4: 0 0 16 8: 3 50 66 16: 0 0 66 32: 0 0 66 64: 0 0 66 128: 0 0 66 256: 2 33 100 mdc.lustre-MDT0001-mdc-ffff9581988df800.batch_stats= snapshot_time: 1778658328.148463161 secs.nsecs start_time: 1778657930.881126765 secs.nsecs elapsed_time: 397.267336396 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 2: 0 0 0 4: 1 11 11 8: 2 22 33 16: 1 11 44 32: 2 22 66 64: 1 11 77 128: 0 0 77 256: 2 22 100 - open/close 732 (time 1778658340.75 total 10.02 last 73.07) - open/close 1486 (time 1778658350.76 total 20.03 last 75.31) - open/close 2262 (time 1778658360.77 total 30.04 last 77.54) - open/close 3085 (time 1778658370.78 total 40.04 last 82.25) - open/close 3852 (time 1778658380.79 total 50.05 last 76.63) - open/close 4591 (time 1778658390.79 total 60.06 last 73.85) - open/close 5357 (time 1778658400.79 total 70.06 last 76.60) - open/close 6089 (time 1778658410.80 total 80.06 last 73.19) - open/close 6890 (time 1778658420.81 total 90.08 last 79.98) - open/close 7603 (time 1778658430.82 total 100.08 last 71.25) - open/close 8318 (time 1778658440.82 total 110.09 last 71.47) - open/close 8947 (time 1778658450.83 total 120.09 last 62.86) - open/close 9648 (time 1778658460.83 total 130.09 last 70.10) total: 10000 open/close in 135.97 seconds: 73.55 ops/second llite.lustre-ffff9581988df800.statahead_batch_max=0 llite.lustre-ffff9581988df800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9581988df800.stats=clear mdc.lustre-MDT0001-mdc-ffff9581988df800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m5.255s user 0m0.306s sys 0m16.649s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9581988df800.statahead_batch_max=256 llite.lustre-ffff9581988df800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9581988df800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9581988df800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9581988df800.stats=clear mdc.lustre-MDT0001-mdc-ffff9581988df800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m54.973s user 0m0.308s sys 0m16.655s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9581988df800.batch_stats= snapshot_time: 1778658609.065688365 secs.nsecs start_time: 1778658541.822418974 secs.nsecs elapsed_time: 67.243269391 secs.nsecs subreqs per batch batches % cum % 1: 285 58 58 2: 61 12 70 4: 49 10 80 8: 26 5 86 16: 11 2 88 32: 16 3 91 64: 6 1 93 128: 1 0 93 256: 33 6 100 mdc.lustre-MDT0001-mdc-ffff9581988df800.batch_stats= snapshot_time: 1778658609.065766315 secs.nsecs start_time: 1778658541.822501747 secs.nsecs elapsed_time: 67.243264568 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9581988df800.statahead_batch_max=64 llite.lustre-ffff9581988df800.statahead_max=128