== sanity test 123ad: Verify batching statahead works correctly ========================================================== 12:05:49 (1787846749) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.batch_stats=0 llite.lustre-ffff8c1512bbe000.statahead_max=32 llite.lustre-ffff8c1512bbe000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.63 seconds: 38.04 ops/second llite.lustre-ffff8c1512bbe000.statahead_max=0 101 real 0m2.590s user 0m0.006s sys 0m0.321s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff8c1512bbe000.statahead_max=32 llite.lustre-ffff8c1512bbe000.statahead_max=32 101 real 0m0.880s user 0m0.002s sys 0m0.148s 'ls -l' 100 files with statahead: 1 sec statahead total: 139 statahead wrong: 2 agl total: 130 list_total: 31 fname_total: 82 hit_total: 3608 miss_total: 1533 swrong=2 ewrong=2 - open/close 616 (time 1787846778.35 total 10.02 last 61.47) total: 900 open/close in 13.71 seconds: 65.66 ops/second llite.lustre-ffff8c1512bbe000.statahead_max=0 1001 real 0m26.763s user 0m0.037s sys 0m2.927s 'ls -l' 1000 files without statahead: 26 sec llite.lustre-ffff8c1512bbe000.statahead_max=32 llite.lustre-ffff8c1512bbe000.statahead_max=32 1001 real 0m9.922s user 0m0.025s sys 0m1.261s 'ls -l' 1000 files with statahead: 10 sec statahead total: 140 statahead wrong: 2 agl total: 131 list_total: 32 fname_total: 82 hit_total: 4605 miss_total: 1536 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 23 seconds llite.lustre-ffff8c1512bbe000.statahead_stats= statahead total: 140 statahead wrong: 2 agl total: 131 list_total: 32 fname_total: 82 hit_total: 4605 miss_total: 1536 mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.batch_stats= snapshot_time: 1787846859.152346764 secs.nsecs start_time: 1787846754.514851162 secs.nsecs elapsed_time: 104.637495602 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 3 7 7 8: 3 7 15 16: 19 48 64 32: 14 35 100 mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.batch_stats= snapshot_time: 1787846859.152407034 secs.nsecs start_time: 1787846754.516457460 secs.nsecs elapsed_time: 104.635949574 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 2 5 5 4: 1 2 7 8: 3 7 15 16: 23 58 74 32: 10 25 100 - open/close 738 (time 1787846872.65 total 10.02 last 73.69) - open/close 1518 (time 1787846882.66 total 20.03 last 77.90) - open/close 2251 (time 1787846892.67 total 30.03 last 73.26) - open/close 3094 (time 1787846902.68 total 40.04 last 84.22) - open/close 3607 (time 1787846912.69 total 50.05 last 51.26) - open/close 4321 (time 1787846922.70 total 60.06 last 71.34) - open/close 5058 (time 1787846932.71 total 70.07 last 73.62) - open/close 5541 (time 1787846942.71 total 80.08 last 48.27) - open/close 6085 (time 1787846952.72 total 90.09 last 54.35) - open/close 6782 (time 1787846962.74 total 100.11 last 69.56) - open/close 7464 (time 1787846972.75 total 110.12 last 68.13) - open/close 8172 (time 1787846982.76 total 120.13 last 70.73) - open/close 8788 (time 1787846992.76 total 130.13 last 61.60) - open/close 9360 (time 1787847002.76 total 140.13 last 57.19) total: 10000 open/close in 148.66 seconds: 67.27 ops/second llite.lustre-ffff8c1512bbe000.statahead_batch_max=0 llite.lustre-ffff8c1512bbe000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.stats=clear mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m10.005s user 0m0.323s sys 0m14.539s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8c1512bbe000.statahead_batch_max=32 llite.lustre-ffff8c1512bbe000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.stats=clear mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m57.862s user 0m0.329s sys 0m13.960s batched statahead RPC= hit count=9969 mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.batch_stats= snapshot_time: 1787847220.297959660 secs.nsecs start_time: 1787847089.312144516 secs.nsecs elapsed_time: 130.985815144 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 2 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.batch_stats= snapshot_time: 1787847220.298160518 secs.nsecs start_time: 1787847089.312303611 secs.nsecs elapsed_time: 130.985856907 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.batch_stats=0 llite.lustre-ffff8c1512bbe000.statahead_max=2048 llite.lustre-ffff8c1512bbe000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.74 seconds: 36.46 ops/second llite.lustre-ffff8c1512bbe000.statahead_max=0 101 real 0m2.127s user 0m0.003s sys 0m0.313s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff8c1512bbe000.statahead_max=2048 llite.lustre-ffff8c1512bbe000.statahead_max=2048 101 real 0m0.844s user 0m0.004s sys 0m0.179s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10068 miss_total: 32 swrong=0 ewrong=0 - open/close 870 (time 1787847511.21 total 10.01 last 86.92) total: 900 open/close in 10.34 seconds: 87.01 ops/second llite.lustre-ffff8c1512bbe000.statahead_max=0 1001 real 0m20.548s user 0m0.046s sys 0m3.058s 'ls -l' 1000 files without statahead: 21 sec llite.lustre-ffff8c1512bbe000.statahead_max=2048 llite.lustre-ffff8c1512bbe000.statahead_max=2048 1001 real 0m6.405s user 0m0.034s sys 0m1.366s 'ls -l' 1000 files with statahead: 6 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11067 miss_total: 33 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 18 seconds llite.lustre-ffff8c1512bbe000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11067 miss_total: 33 mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.batch_stats= snapshot_time: 1787847568.841477451 secs.nsecs start_time: 1787847222.735275430 secs.nsecs elapsed_time: 346.106202021 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 1 14 14 4: 2 28 42 8: 2 28 71 16: 0 0 71 32: 0 0 71 64: 0 0 71 128: 0 0 71 256: 2 28 100 mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.batch_stats= snapshot_time: 1787847568.842896406 secs.nsecs start_time: 1787847222.735805165 secs.nsecs elapsed_time: 346.107091241 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 18 18 8: 1 9 27 16: 2 18 45 32: 4 36 81 64: 0 0 81 128: 0 0 81 256: 2 18 100 - open/close 746 (time 1787847581.88 total 10.00 last 74.57) - open/close 1425 (time 1787847591.88 total 20.01 last 67.89) - open/close 2061 (time 1787847601.89 total 30.01 last 63.58) - open/close 2727 (time 1787847611.90 total 40.02 last 66.54) - open/close 3381 (time 1787847621.90 total 50.02 last 65.40) - open/close 3950 (time 1787847631.90 total 60.02 last 56.90) - open/close 4826 (time 1787847641.90 total 70.02 last 87.56) - open/close 5583 (time 1787847651.90 total 80.02 last 75.69) - open/close 6307 (time 1787847661.91 total 90.03 last 72.33) - open/close 7139 (time 1787847671.91 total 100.04 last 83.19) - open/close 7918 (time 1787847681.92 total 110.04 last 77.85) - open/close 8591 (time 1787847691.92 total 120.05 last 67.27) - open/close 9350 (time 1787847701.93 total 130.05 last 75.86) total: 10000 open/close in 139.10 seconds: 71.89 ops/second llite.lustre-ffff8c1512bbe000.statahead_batch_max=0 llite.lustre-ffff8c1512bbe000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.stats=clear mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m0.067s user 0m0.339s sys 0m15.271s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8c1512bbe000.statahead_batch_max=256 llite.lustre-ffff8c1512bbe000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.stats=clear mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m53.050s user 0m0.292s sys 0m14.474s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff8c1512bbe000.batch_stats= snapshot_time: 1787847845.563977439 secs.nsecs start_time: 1787847781.047240355 secs.nsecs elapsed_time: 64.516737084 secs.nsecs subreqs per batch batches %% cum %% 1: 100 42 42 2: 35 15 57 4: 19 8 66 8: 12 5 71 16: 6 2 73 32: 16 6 80 64: 10 4 84 128: 1 0 85 256: 34 14 100 mdc.lustre-MDT0001-mdc-ffff8c1512bbe000.batch_stats= snapshot_time: 1787847845.566417573 secs.nsecs start_time: 1787847781.047394962 secs.nsecs elapsed_time: 64.519022611 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff8c1512bbe000.statahead_batch_max=64 llite.lustre-ffff8c1512bbe000.statahead_max=128