== sanity test 123ad: Verify batching statahead works correctly ========================================================== 21:33:29 (1746149609) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff880130985800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff880130985800.batch_stats=0 llite.lustre-ffff880130985800.statahead_max=32 llite.lustre-ffff880130985800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff880130985800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff880130985800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.39 seconds: 41.80 ops/second llite.lustre-ffff880130985800.statahead_max=0 101 real 0m2.097s user 0m0.019s sys 0m0.427s ls -l 100 files without statahead: 2 sec llite.lustre-ffff880130985800.statahead_max=32 32 101 real 0m0.689s user 0m0.011s sys 0m0.212s ls -l 100 files with statahead: 1 sec statahead total: 118 statahead wrong: 2 agl total: 118 list_total: 28 fname_total: 90 hit_total: 4138 miss_total: 1325 - open/close 866 (time 1746149632.81 total 10.00 last 86.58) total: 900 open/close in 10.35 seconds: 86.97 ops/second llite.lustre-ffff880130985800.statahead_max=0 1001 real 0m18.943s user 0m0.079s sys 0m3.862s ls -l 1000 files without statahead: 19 sec llite.lustre-ffff880130985800.statahead_max=32 32 1001 real 0m5.615s user 0m0.050s sys 0m1.692s ls -l 1000 files with statahead: 6 sec statahead total: 119 statahead wrong: 2 agl total: 119 list_total: 29 fname_total: 90 hit_total: 5137 miss_total: 1328 - open/close 763 (time 1746149673.26 total 10.02 last 76.16) total: 1000 open/close in 13.13 seconds: 76.15 ops/second llite.lustre-ffff880130985800.statahead_max=0 2001 real 0m39.256s user 0m0.189s sys 0m7.649s ls -l 2000 files without statahead: 39 sec llite.lustre-ffff880130985800.statahead_max=32 32 2001 real 0m12.959s user 0m0.122s sys 0m3.440s ls -l 2000 files with statahead: 12 sec statahead total: 120 statahead wrong: 2 agl total: 120 list_total: 30 fname_total: 90 hit_total: 7136 miss_total: 1341 ls -l done rm -r /mnt/lustre/d123ad.sanity: 33 seconds llite.lustre-ffff880130985800.statahead_stats= statahead total: 120 statahead wrong: 2 agl total: 120 list_total: 30 fname_total: 90 hit_total: 7136 miss_total: 1341 mdc.lustre-MDT0000-mdc-ffff880130985800.batch_stats= snapshot_time: 1746149769.653793854 secs.nsecs start_time: 1746149613.158989834 secs.nsecs elapsed_time: 156.494804020 secs.nsecs subreqs per batch batches % cum % 1: 2 1 1 2: 1 0 2 4: 5 4 7 8: 2 1 9 16: 71 69 79 32: 21 20 100 mdc.lustre-MDT0001-mdc-ffff880130985800.batch_stats= snapshot_time: 1746149769.654015847 secs.nsecs start_time: 1746149613.159424887 secs.nsecs elapsed_time: 156.494590960 secs.nsecs subreqs per batch batches % cum % 1: 1 0 0 2: 0 0 0 4: 2 1 2 8: 5 4 7 16: 30 28 36 32: 66 63 100 - open/close 755 (time 1746149781.66 total 10.03 last 75.31) - open/close 1558 (time 1746149791.67 total 20.04 last 80.18) - open/close 2386 (time 1746149801.68 total 30.05 last 82.73) - open/close 3286 (time 1746149811.68 total 40.05 last 89.96) - open/close 4095 (time 1746149821.69 total 50.06 last 80.87) - open/close 4861 (time 1746149831.70 total 60.07 last 76.50) - open/close 5673 (time 1746149841.71 total 70.08 last 81.14) - open/close 6444 (time 1746149851.71 total 80.08 last 77.08) - open/close 7268 (time 1746149861.72 total 90.09 last 82.29) - open/close 8254 (time 1746149871.73 total 100.10 last 98.56) - open/close 9053 (time 1746149881.74 total 110.11 last 79.78) - open/close 9835 (time 1746149891.75 total 120.12 last 78.14) total: 10000 open/close in 122.21 seconds: 81.82 ops/second llite.lustre-ffff880130985800.statahead_batch_max=0 llite.lustre-ffff880130985800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff880130985800.stats=clear mdc.lustre-MDT0001-mdc-ffff880130985800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m37.992s user 0m0.386s sys 0m14.009s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff880130985800.statahead_batch_max=32 llite.lustre-ffff880130985800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff880130985800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff880130985800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff880130985800.stats=clear mdc.lustre-MDT0001-mdc-ffff880130985800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m20.458s user 0m0.465s sys 0m16.467s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff880130985800.batch_stats= snapshot_time: 1746150025.636877538 secs.nsecs start_time: 1746149937.319879967 secs.nsecs elapsed_time: 88.316997571 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 2 0 0 32: 312 99 100 mdc.lustre-MDT0001-mdc-ffff880130985800.batch_stats= snapshot_time: 1746150025.637147371 secs.nsecs start_time: 1746149937.319967752 secs.nsecs elapsed_time: 88.317179619 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff880130985800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff880130985800.batch_stats=0 llite.lustre-ffff880130985800.statahead_max=2048 llite.lustre-ffff880130985800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff880130985800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff880130985800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.53 seconds: 39.53 ops/second llite.lustre-ffff880130985800.statahead_max=0 101 real 0m2.294s user 0m0.016s sys 0m0.441s ls -l 100 files without statahead: 3 sec llite.lustre-ffff880130985800.statahead_max=2048 2048 101 real 0m0.787s user 0m0.008s sys 0m0.211s ls -l 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10098 miss_total: 86 - open/close 743 (time 1746150307.35 total 10.01 last 74.21) total: 900 open/close in 12.18 seconds: 73.87 ops/second llite.lustre-ffff880130985800.statahead_max=0 1001 real 0m19.634s user 0m0.064s sys 0m3.890s ls -l 1000 files without statahead: 20 sec llite.lustre-ffff880130985800.statahead_max=2048 2048 1001 real 0m3.743s user 0m0.038s sys 0m1.593s ls -l 1000 files with statahead: 3 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11097 miss_total: 87 - open/close 757 (time 1746150349.65 total 10.01 last 75.63) total: 1000 open/close in 13.49 seconds: 74.12 ops/second llite.lustre-ffff880130985800.statahead_max=0 2001 real 0m37.955s user 0m0.218s sys 0m7.602s ls -l 2000 files without statahead: 38 sec llite.lustre-ffff880130985800.statahead_max=2048 2048 2001 real 0m6.121s user 0m0.084s sys 0m3.219s ls -l 2000 files with statahead: 6 sec statahead total: 4 statahead wrong: 0 agl total: 4 list_total: 4 fname_total: 0 hit_total: 13096 miss_total: 88 ls -l done rm -r /mnt/lustre/d123ad.sanity: 29 seconds llite.lustre-ffff880130985800.statahead_stats= statahead total: 4 statahead wrong: 0 agl total: 4 list_total: 4 fname_total: 0 hit_total: 13096 miss_total: 88 mdc.lustre-MDT0000-mdc-ffff880130985800.batch_stats= snapshot_time: 1746150432.960323821 secs.nsecs start_time: 1746150027.134448873 secs.nsecs elapsed_time: 405.825874948 secs.nsecs subreqs per batch batches % cum % 1: 2 14 14 2: 0 0 14 4: 4 28 42 8: 1 7 50 16: 0 0 50 32: 1 7 57 64: 0 0 57 128: 0 0 57 256: 6 42 100 mdc.lustre-MDT0001-mdc-ffff880130985800.batch_stats= snapshot_time: 1746150432.960451484 secs.nsecs start_time: 1746150027.134558670 secs.nsecs elapsed_time: 405.825892814 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 2: 0 0 0 4: 2 12 12 8: 5 31 43 16: 0 0 43 32: 1 6 50 64: 1 6 56 128: 1 6 62 256: 6 37 100 - open/close 753 (time 1746150444.47 total 10.00 last 75.29) - open/close 1434 (time 1746150454.48 total 20.01 last 68.01) - open/close 2167 (time 1746150464.49 total 30.02 last 73.24) - open/close 2885 (time 1746150474.49 total 40.03 last 71.78) - open/close 3583 (time 1746150484.50 total 50.03 last 69.75) - open/close 4435 (time 1746150494.50 total 60.03 last 85.18) - open/close 5295 (time 1746150504.51 total 70.04 last 85.91) - open/close 6106 (time 1746150514.53 total 80.06 last 80.99) - open/close 6894 (time 1746150524.53 total 90.06 last 78.75) - open/close 7777 (time 1746150534.54 total 100.07 last 88.25) - open/close 8891 (time 1746150544.55 total 110.08 last 111.30) - open/close 9756 (time 1746150554.56 total 120.09 last 86.43) total: 10000 open/close in 122.99 seconds: 81.31 ops/second llite.lustre-ffff880130985800.statahead_batch_max=0 llite.lustre-ffff880130985800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff880130985800.stats=clear mdc.lustre-MDT0001-mdc-ffff880130985800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m26.509s user 0m0.408s sys 0m13.435s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff880130985800.statahead_batch_max=256 llite.lustre-ffff880130985800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff880130985800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff880130985800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff880130985800.stats=clear mdc.lustre-MDT0001-mdc-ffff880130985800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m25.763s user 0m0.430s sys 0m15.052s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff880130985800.batch_stats= snapshot_time: 1746150619.266481654 secs.nsecs start_time: 1746150588.796595166 secs.nsecs elapsed_time: 30.469886488 secs.nsecs subreqs per batch batches % cum % 1: 132 54 54 2: 13 5 59 4: 16 6 66 8: 13 5 71 16: 16 6 78 32: 11 4 83 64: 5 2 85 128: 1 0 85 256: 35 14 100 mdc.lustre-MDT0001-mdc-ffff880130985800.batch_stats= snapshot_time: 1746150619.266592767 secs.nsecs start_time: 1746150588.797490215 secs.nsecs elapsed_time: 30.469102552 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff880130985800.statahead_batch_max=64 llite.lustre-ffff880130985800.statahead_max=128