== sanity test 123ad: Verify batching statahead works correctly ========================================================== 06:28:39 (1787826519) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9ead7684f800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9ead7684f800.batch_stats=0 llite.lustre-ffff9ead7684f800.statahead_max=32 llite.lustre-ffff9ead7684f800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9ead7684f800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9ead7684f800.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.48 seconds: 40.37 ops/second llite.lustre-ffff9ead7684f800.statahead_max=0 101 real 0m2.428s user 0m0.006s sys 0m0.357s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff9ead7684f800.statahead_max=32 llite.lustre-ffff9ead7684f800.statahead_max=32 101 real 0m1.126s user 0m0.001s sys 0m0.186s 'ls -l' 100 files with statahead: 2 sec statahead total: 135 statahead wrong: 2 agl total: 129 list_total: 30 fname_total: 79 hit_total: 3596 miss_total: 1528 swrong=2 ewrong=2 - open/close 626 (time 1787826550.15 total 10.03 last 62.44) total: 900 open/close in 13.82 seconds: 65.10 ops/second llite.lustre-ffff9ead7684f800.statahead_max=0 1001 real 0m25.131s user 0m0.039s sys 0m3.259s 'ls -l' 1000 files without statahead: 25 sec llite.lustre-ffff9ead7684f800.statahead_max=32 llite.lustre-ffff9ead7684f800.statahead_max=32 1001 real 0m10.051s user 0m0.035s sys 0m1.394s 'ls -l' 1000 files with statahead: 10 sec statahead total: 136 statahead wrong: 2 agl total: 130 list_total: 31 fname_total: 79 hit_total: 4594 miss_total: 1530 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 23 seconds llite.lustre-ffff9ead7684f800.statahead_stats= statahead total: 136 statahead wrong: 2 agl total: 130 list_total: 31 fname_total: 79 hit_total: 4594 miss_total: 1530 mdc.lustre-MDT0000-mdc-ffff9ead7684f800.batch_stats= snapshot_time: 1787826626.213766695 secs.nsecs start_time: 1787826525.621513200 secs.nsecs elapsed_time: 100.592253495 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 3 7 10 8: 1 2 12 16: 23 58 71 32: 11 28 100 mdc.lustre-MDT0001-mdc-ffff9ead7684f800.batch_stats= snapshot_time: 1787826626.213849768 secs.nsecs start_time: 1787826525.621610096 secs.nsecs elapsed_time: 100.592239672 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 1 2 2 4: 2 5 7 8: 4 10 17 16: 21 53 71 32: 11 28 100 - open/close 690 (time 1787826639.44 total 10.00 last 69.00) - open/close 1512 (time 1787826649.44 total 20.01 last 82.16) - open/close 2392 (time 1787826659.44 total 30.01 last 87.99) - open/close 3184 (time 1787826669.45 total 40.02 last 79.13) - open/close 4002 (time 1787826679.46 total 50.02 last 81.78) - open/close 4708 (time 1787826689.46 total 60.03 last 70.54) - open/close 5491 (time 1787826699.49 total 70.05 last 78.11) - open/close 6267 (time 1787826709.50 total 80.06 last 77.54) - open/close 7034 (time 1787826719.50 total 90.06 last 76.67) - open/close 7603 (time 1787826729.50 total 100.07 last 56.88) - open/close 8401 (time 1787826739.53 total 110.09 last 79.61) - open/close 9054 (time 1787826749.53 total 120.10 last 65.26) - open/close 9726 (time 1787826759.54 total 130.10 last 67.17) total: 10000 open/close in 134.37 seconds: 74.42 ops/second llite.lustre-ffff9ead7684f800.statahead_batch_max=0 llite.lustre-ffff9ead7684f800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9ead7684f800.stats=clear mdc.lustre-MDT0001-mdc-ffff9ead7684f800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m10.449s user 0m0.369s sys 0m15.397s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9ead7684f800.statahead_batch_max=32 llite.lustre-ffff9ead7684f800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9ead7684f800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9ead7684f800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9ead7684f800.stats=clear mdc.lustre-MDT0001-mdc-ffff9ead7684f800.stats=clear ls -l files in d123ad.sanity: 10001 real 2m1.985s user 0m0.358s sys 0m14.638s batched statahead RPC= hit count=9979 mdc.lustre-MDT0000-mdc-ffff9ead7684f800.batch_stats= snapshot_time: 1787826978.259506538 secs.nsecs start_time: 1787826844.789054121 secs.nsecs elapsed_time: 133.470452417 secs.nsecs subreqs per batch batches %% cum %% 1: 1 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9ead7684f800.batch_stats= snapshot_time: 1787826978.259594744 secs.nsecs start_time: 1787826844.789152126 secs.nsecs elapsed_time: 133.470442618 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9ead7684f800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9ead7684f800.batch_stats=0 llite.lustre-ffff9ead7684f800.statahead_max=2048 llite.lustre-ffff9ead7684f800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9ead7684f800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9ead7684f800.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.88 seconds: 34.76 ops/second llite.lustre-ffff9ead7684f800.statahead_max=0 101 real 0m2.409s user 0m0.008s sys 0m0.360s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9ead7684f800.statahead_max=2048 llite.lustre-ffff9ead7684f800.statahead_max=2048 101 real 0m0.847s user 0m0.005s sys 0m0.199s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10078 miss_total: 22 swrong=0 ewrong=0 - open/close 748 (time 1787827308.19 total 10.01 last 74.70) total: 900 open/close in 12.56 seconds: 71.66 ops/second llite.lustre-ffff9ead7684f800.statahead_max=0 1001 real 0m25.643s user 0m0.049s sys 0m3.285s 'ls -l' 1000 files without statahead: 26 sec llite.lustre-ffff9ead7684f800.statahead_max=2048 llite.lustre-ffff9ead7684f800.statahead_max=2048 1001 real 0m7.306s user 0m0.037s sys 0m1.521s 'ls -l' 1000 files with statahead: 7 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11077 miss_total: 23 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 25 seconds llite.lustre-ffff9ead7684f800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11077 miss_total: 23 mdc.lustre-MDT0000-mdc-ffff9ead7684f800.batch_stats= snapshot_time: 1787827382.658549485 secs.nsecs start_time: 1787826981.088841608 secs.nsecs elapsed_time: 401.569707877 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 3 33 33 8: 1 11 44 16: 1 11 55 32: 1 11 66 64: 1 11 77 128: 0 0 77 256: 2 22 100 mdc.lustre-MDT0001-mdc-ffff9ead7684f800.batch_stats= snapshot_time: 1787827382.658623215 secs.nsecs start_time: 1787826981.089012881 secs.nsecs elapsed_time: 401.569610334 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 1 11 11 4: 1 11 22 8: 2 22 44 16: 0 0 44 32: 2 22 66 64: 1 11 77 128: 0 0 77 256: 2 22 100 - open/close 643 (time 1787827395.92 total 10.02 last 64.18) - open/close 1324 (time 1787827405.94 total 20.03 last 68.00) - open/close 2158 (time 1787827415.95 total 30.04 last 83.32) - open/close 2837 (time 1787827425.96 total 40.05 last 67.85) - open/close 3486 (time 1787827435.96 total 50.06 last 64.85) - open/close 4309 (time 1787827445.96 total 60.06 last 82.29) - open/close 5223 (time 1787827455.97 total 70.06 last 91.35) - open/close 5902 (time 1787827465.97 total 80.06 last 67.90) - open/close 6629 (time 1787827475.98 total 90.07 last 72.63) - open/close 7221 (time 1787827486.00 total 100.10 last 59.07) - open/close 7754 (time 1787827496.00 total 110.10 last 53.29) - open/close 8560 (time 1787827506.01 total 120.11 last 80.52) - open/close 9211 (time 1787827516.04 total 130.14 last 64.93) - open/close 9703 (time 1787827526.04 total 140.14 last 49.19) total: 10000 open/close in 144.27 seconds: 69.32 ops/second llite.lustre-ffff9ead7684f800.statahead_batch_max=0 llite.lustre-ffff9ead7684f800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9ead7684f800.stats=clear mdc.lustre-MDT0001-mdc-ffff9ead7684f800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m14.101s user 0m0.396s sys 0m15.603s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9ead7684f800.statahead_batch_max=256 llite.lustre-ffff9ead7684f800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9ead7684f800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9ead7684f800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9ead7684f800.stats=clear mdc.lustre-MDT0001-mdc-ffff9ead7684f800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m6.651s user 0m0.359s sys 0m14.905s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9ead7684f800.batch_stats= snapshot_time: 1787827692.833871151 secs.nsecs start_time: 1787827614.658794395 secs.nsecs elapsed_time: 78.175076756 secs.nsecs subreqs per batch batches %% cum %% 1: 134 46 46 2: 38 13 60 4: 23 8 68 8: 25 8 76 16: 10 3 80 32: 13 4 84 64: 8 2 87 128: 1 0 88 256: 34 11 100 mdc.lustre-MDT0001-mdc-ffff9ead7684f800.batch_stats= snapshot_time: 1787827692.833952980 secs.nsecs start_time: 1787827614.660582604 secs.nsecs elapsed_time: 78.173370376 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9ead7684f800.statahead_batch_max=64 llite.lustre-ffff9ead7684f800.statahead_max=128