== sanity test 123ad: Verify batching statahead works correctly ========================================================== 03:54:46 (1786953286) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.batch_stats=0 llite.lustre-ffff9d7453ca6800.statahead_max=32 llite.lustre-ffff9d7453ca6800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.88 seconds: 34.71 ops/second llite.lustre-ffff9d7453ca6800.statahead_max=0 101 real 0m2.309s user 0m0.005s sys 0m0.347s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9d7453ca6800.statahead_max=32 llite.lustre-ffff9d7453ca6800.statahead_max=32 101 real 0m1.449s user 0m0.006s sys 0m0.183s 'ls -l' 100 files with statahead: 2 sec statahead total: 137 statahead wrong: 2 agl total: 131 list_total: 31 fname_total: 105 hit_total: 3607 miss_total: 1320 swrong=2 ewrong=2 - open/close 693 (time 1786953317.52 total 10.01 last 69.24) total: 900 open/close in 12.84 seconds: 70.09 ops/second llite.lustre-ffff9d7453ca6800.statahead_max=0 1001 real 0m26.258s user 0m0.039s sys 0m3.456s 'ls -l' 1000 files without statahead: 26 sec llite.lustre-ffff9d7453ca6800.statahead_max=32 llite.lustre-ffff9d7453ca6800.statahead_max=32 1001 real 0m11.766s user 0m0.028s sys 0m1.681s 'ls -l' 1000 files with statahead: 11 sec statahead total: 138 statahead wrong: 2 agl total: 132 list_total: 32 fname_total: 105 hit_total: 4604 miss_total: 1323 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 22 seconds llite.lustre-ffff9d7453ca6800.statahead_stats= statahead total: 138 statahead wrong: 2 agl total: 132 list_total: 32 fname_total: 105 hit_total: 4604 miss_total: 1323 mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.batch_stats= snapshot_time: 1786953395.594988263 secs.nsecs start_time: 1786953292.629832781 secs.nsecs elapsed_time: 102.965155482 secs.nsecs subreqs per batch batches %% cum %% 1: 2 5 5 2: 2 5 10 4: 2 5 16 8: 1 2 18 16: 25 67 86 32: 5 13 100 mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.batch_stats= snapshot_time: 1786953395.595096003 secs.nsecs start_time: 1786953292.629908153 secs.nsecs elapsed_time: 102.965187850 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 1 2 5 8: 3 7 12 16: 14 35 47 32: 21 52 100 - open/close 684 (time 1786953408.75 total 10.02 last 68.28) - open/close 1202 (time 1786953418.75 total 20.02 last 51.77) - open/close 1849 (time 1786953428.77 total 30.04 last 64.61) - open/close 2587 (time 1786953438.78 total 40.05 last 73.71) - open/close 3238 (time 1786953448.78 total 50.05 last 65.09) - open/close 3871 (time 1786953458.79 total 60.06 last 63.25) - open/close 4602 (time 1786953468.80 total 70.07 last 73.03) - open/close 5332 (time 1786953478.80 total 80.07 last 72.95) - open/close 5976 (time 1786953488.82 total 90.09 last 64.31) - open/close 6694 (time 1786953498.82 total 100.09 last 71.78) - open/close 7313 (time 1786953508.83 total 110.10 last 61.83) - open/close 7943 (time 1786953518.84 total 120.11 last 62.97) - open/close 8506 (time 1786953528.85 total 130.12 last 56.25) - open/close 9112 (time 1786953538.86 total 140.13 last 60.53) - open/close 9737 (time 1786953548.86 total 150.13 last 62.48) total: 10000 open/close in 154.09 seconds: 64.90 ops/second llite.lustre-ffff9d7453ca6800.statahead_batch_max=0 llite.lustre-ffff9d7453ca6800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.stats=clear mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m18.539s user 0m0.368s sys 0m18.791s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9d7453ca6800.statahead_batch_max=32 llite.lustre-ffff9d7453ca6800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.stats=clear mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.stats=clear ls -l files in d123ad.sanity: 10001 real 2m6.887s user 0m0.388s sys 0m17.601s batched statahead RPC= hit count=9985 mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.batch_stats= snapshot_time: 1786953784.100405161 secs.nsecs start_time: 1786953643.321661838 secs.nsecs elapsed_time: 140.778743323 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.batch_stats= snapshot_time: 1786953784.100474441 secs.nsecs start_time: 1786953643.321896767 secs.nsecs elapsed_time: 140.778577674 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.batch_stats=0 llite.lustre-ffff9d7453ca6800.statahead_max=2048 llite.lustre-ffff9d7453ca6800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.29 seconds: 30.38 ops/second llite.lustre-ffff9d7453ca6800.statahead_max=0 101 real 0m3.692s user 0m0.006s sys 0m0.381s 'ls -l' 100 files without statahead: 4 sec llite.lustre-ffff9d7453ca6800.statahead_max=2048 llite.lustre-ffff9d7453ca6800.statahead_max=2048 101 real 0m1.245s user 0m0.007s sys 0m0.181s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10084 miss_total: 16 swrong=0 ewrong=0 - open/close 707 (time 1786954170.69 total 10.01 last 70.62) total: 900 open/close in 12.84 seconds: 70.11 ops/second llite.lustre-ffff9d7453ca6800.statahead_max=0 1001 real 0m28.255s user 0m0.040s sys 0m3.438s 'ls -l' 1000 files without statahead: 28 sec llite.lustre-ffff9d7453ca6800.statahead_max=2048 llite.lustre-ffff9d7453ca6800.statahead_max=2048 1001 real 0m8.392s user 0m0.034s sys 0m1.690s 'ls -l' 1000 files with statahead: 8 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11083 miss_total: 17 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 24 seconds llite.lustre-ffff9d7453ca6800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11083 miss_total: 17 mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.batch_stats= snapshot_time: 1786954250.194726126 secs.nsecs start_time: 1786953787.617495517 secs.nsecs elapsed_time: 462.577230609 secs.nsecs subreqs per batch batches %% cum %% 1: 1 14 14 2: 2 28 42 4: 1 14 57 8: 1 14 71 16: 0 0 71 32: 0 0 71 64: 0 0 71 128: 0 0 71 256: 2 28 100 mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.batch_stats= snapshot_time: 1786954250.194868444 secs.nsecs start_time: 1786953787.619475894 secs.nsecs elapsed_time: 462.575392550 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 3 30 30 16: 3 30 60 32: 1 10 70 64: 1 10 80 128: 0 0 80 256: 2 20 100 - open/close 606 (time 1786954263.36 total 10.00 last 60.58) - open/close 1256 (time 1786954273.37 total 20.01 last 64.94) - open/close 1966 (time 1786954283.37 total 30.01 last 71.00) - open/close 2666 (time 1786954293.38 total 40.02 last 69.98) - open/close 3402 (time 1786954303.38 total 50.02 last 73.58) - open/close 4075 (time 1786954313.39 total 60.03 last 67.25) - open/close 4757 (time 1786954323.39 total 70.03 last 68.18) - open/close 5460 (time 1786954333.40 total 80.04 last 70.24) - open/close 6213 (time 1786954343.40 total 90.04 last 75.27) - open/close 6893 (time 1786954353.41 total 100.05 last 67.95) - open/close 7634 (time 1786954363.41 total 110.05 last 74.07) - open/close 8395 (time 1786954373.42 total 120.06 last 76.05) - open/close 9008 (time 1786954383.44 total 130.07 last 61.21) - open/close 9701 (time 1786954393.44 total 140.08 last 69.25) total: 10000 open/close in 144.91 seconds: 69.01 ops/second llite.lustre-ffff9d7453ca6800.statahead_batch_max=0 llite.lustre-ffff9d7453ca6800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.stats=clear mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m9.075s user 0m0.361s sys 0m19.444s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9d7453ca6800.statahead_batch_max=256 llite.lustre-ffff9d7453ca6800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.stats=clear mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m1.251s user 0m0.364s sys 0m18.695s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9d7453ca6800.batch_stats= snapshot_time: 1786954552.412490892 secs.nsecs start_time: 1786954477.915816887 secs.nsecs elapsed_time: 74.496674005 secs.nsecs subreqs per batch batches %% cum %% 1: 284 54 54 2: 82 15 70 4: 55 10 80 8: 29 5 86 16: 18 3 90 32: 16 3 93 64: 3 0 93 128: 0 0 93 256: 33 6 100 mdc.lustre-MDT0001-mdc-ffff9d7453ca6800.batch_stats= snapshot_time: 1786954552.412573126 secs.nsecs start_time: 1786954477.915927749 secs.nsecs elapsed_time: 74.496645377 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9d7453ca6800.statahead_batch_max=64 llite.lustre-ffff9d7453ca6800.statahead_max=128