== sanity test 123ad: Verify batching statahead works correctly ========================================================== 17:37:07 (1790977027) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.batch_stats=0 llite.lustre-ffff9c6ce7db4000.statahead_max=32 llite.lustre-ffff9c6ce7db4000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.33 seconds: 29.99 ops/second llite.lustre-ffff9c6ce7db4000.statahead_max=0 101 real 0m2.481s user 0m0.003s sys 0m0.318s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9c6ce7db4000.statahead_max=32 llite.lustre-ffff9c6ce7db4000.statahead_max=32 101 real 0m1.447s user 0m0.008s sys 0m0.190s 'ls -l' 100 files with statahead: 2 sec statahead total: 152 statahead wrong: 2 agl total: 146 list_total: 30 fname_total: 108 hit_total: 3802 miss_total: 1492 swrong=2 ewrong=2 - open/close 740 (time 1790977057.88 total 10.01 last 73.95) total: 900 open/close in 12.63 seconds: 71.27 ops/second llite.lustre-ffff9c6ce7db4000.statahead_max=0 1001 real 0m25.977s user 0m0.031s sys 0m2.998s 'ls -l' 1000 files without statahead: 26 sec llite.lustre-ffff9c6ce7db4000.statahead_max=32 llite.lustre-ffff9c6ce7db4000.statahead_max=32 1001 real 0m11.600s user 0m0.042s sys 0m1.462s 'ls -l' 1000 files with statahead: 11 sec statahead total: 153 statahead wrong: 2 agl total: 147 list_total: 31 fname_total: 108 hit_total: 4796 miss_total: 1498 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 23 seconds llite.lustre-ffff9c6ce7db4000.statahead_stats= statahead total: 153 statahead wrong: 2 agl total: 147 list_total: 31 fname_total: 108 hit_total: 4796 miss_total: 1498 mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.batch_stats= snapshot_time: 1790977136.022443985 secs.nsecs start_time: 1790977033.038409243 secs.nsecs elapsed_time: 102.984034742 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 5 5 8: 5 12 17 16: 19 48 66 32: 13 33 100 mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.batch_stats= snapshot_time: 1790977136.023246727 secs.nsecs start_time: 1790977033.038483765 secs.nsecs elapsed_time: 102.984762962 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 2 5 7 8: 2 5 12 16: 18 46 58 32: 16 41 100 - open/close 626 (time 1790977148.93 total 10.01 last 62.57) - open/close 1323 (time 1790977158.93 total 20.01 last 69.69) - open/close 1980 (time 1790977168.93 total 30.01 last 65.69) - open/close 2645 (time 1790977178.95 total 40.03 last 66.37) - open/close 3297 (time 1790977188.95 total 50.03 last 65.16) - open/close 3946 (time 1790977198.96 total 60.04 last 64.87) - open/close 4525 (time 1790977208.97 total 70.05 last 57.84) - open/close 5129 (time 1790977218.97 total 80.05 last 60.37) - open/close 5772 (time 1790977228.97 total 90.06 last 64.29) - open/close 6459 (time 1790977238.98 total 100.06 last 68.68) - open/close 7216 (time 1790977248.99 total 110.07 last 75.62) - open/close 7987 (time 1790977258.99 total 120.08 last 77.05) - open/close 8719 (time 1790977269.00 total 130.08 last 73.19) - open/close 9436 (time 1790977279.00 total 140.08 last 71.65) total: 10000 open/close in 147.23 seconds: 67.92 ops/second llite.lustre-ffff9c6ce7db4000.statahead_batch_max=0 llite.lustre-ffff9c6ce7db4000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.stats=clear mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m5.238s user 0m0.339s sys 0m15.584s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9c6ce7db4000.statahead_batch_max=32 llite.lustre-ffff9c6ce7db4000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.stats=clear mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m54.878s user 0m0.338s sys 0m14.967s batched statahead RPC= hit count=9976 mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.batch_stats= snapshot_time: 1790977485.792649077 secs.nsecs start_time: 1790977359.474157611 secs.nsecs elapsed_time: 126.318491466 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 1 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.batch_stats= snapshot_time: 1790977485.793144015 secs.nsecs start_time: 1790977359.474828625 secs.nsecs elapsed_time: 126.318315390 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.batch_stats=0 llite.lustre-ffff9c6ce7db4000.statahead_max=2048 llite.lustre-ffff9c6ce7db4000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.35 seconds: 29.87 ops/second llite.lustre-ffff9c6ce7db4000.statahead_max=0 101 real 0m3.699s user 0m0.005s sys 0m0.352s 'ls -l' 100 files without statahead: 4 sec llite.lustre-ffff9c6ce7db4000.statahead_max=2048 llite.lustre-ffff9c6ce7db4000.statahead_max=2048 101 real 0m0.868s user 0m0.005s sys 0m0.167s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10075 miss_total: 25 swrong=0 ewrong=0 - open/close 688 (time 1790977871.40 total 10.01 last 68.72) total: 900 open/close in 13.77 seconds: 65.37 ops/second llite.lustre-ffff9c6ce7db4000.statahead_max=0 1001 real 0m28.052s user 0m0.039s sys 0m3.027s 'ls -l' 1000 files without statahead: 28 sec llite.lustre-ffff9c6ce7db4000.statahead_max=2048 llite.lustre-ffff9c6ce7db4000.statahead_max=2048 1001 real 0m8.072s user 0m0.017s sys 0m1.388s 'ls -l' 1000 files with statahead: 8 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11074 miss_total: 26 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 26 seconds llite.lustre-ffff9c6ce7db4000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11074 miss_total: 26 mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.batch_stats= snapshot_time: 1790977953.009584709 secs.nsecs start_time: 1790977488.766848495 secs.nsecs elapsed_time: 464.242736214 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 25 25 8: 3 37 62 16: 0 0 62 32: 0 0 62 64: 1 12 75 128: 0 0 75 256: 2 25 100 mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.batch_stats= snapshot_time: 1790977953.010368024 secs.nsecs start_time: 1790977488.767022833 secs.nsecs elapsed_time: 464.243345191 secs.nsecs subreqs per batch batches %% cum %% 1: 1 12 12 2: 0 0 12 4: 1 12 25 8: 2 25 50 16: 1 12 62 32: 0 0 62 64: 1 12 75 128: 0 0 75 256: 2 25 100 - open/close 712 (time 1790977966.01 total 10.01 last 71.16) - open/close 1495 (time 1790977976.02 total 20.01 last 78.26) - open/close 2201 (time 1790977986.03 total 30.03 last 70.50) - open/close 2920 (time 1790977996.05 total 40.04 last 71.79) - open/close 3617 (time 1790978006.05 total 50.05 last 69.67) - open/close 4193 (time 1790978016.06 total 60.06 last 57.54) - open/close 4722 (time 1790978026.07 total 70.07 last 52.84) - open/close 5327 (time 1790978036.08 total 80.08 last 60.44) - open/close 5851 (time 1790978046.09 total 90.08 last 52.37) - open/close 6406 (time 1790978056.09 total 100.09 last 55.47) - open/close 6974 (time 1790978066.10 total 110.09 last 56.77) - open/close 7583 (time 1790978076.11 total 120.11 last 60.81) - open/close 8206 (time 1790978086.14 total 130.13 last 62.17) - open/close 8821 (time 1790978096.14 total 140.14 last 61.45) - open/close 9434 (time 1790978106.15 total 150.15 last 61.24) total: 10000 open/close in 158.92 seconds: 62.93 ops/second llite.lustre-ffff9c6ce7db4000.statahead_batch_max=0 llite.lustre-ffff9c6ce7db4000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.stats=clear mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m30.584s user 0m0.326s sys 0m15.293s unbatched statahead RPC=10005 hit count=9998 llite.lustre-ffff9c6ce7db4000.statahead_batch_max=256 llite.lustre-ffff9c6ce7db4000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.stats=clear mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m58.156s user 0m0.289s sys 0m15.168s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9c6ce7db4000.batch_stats= snapshot_time: 1790978283.708270378 secs.nsecs start_time: 1790978213.502765453 secs.nsecs elapsed_time: 70.205504925 secs.nsecs subreqs per batch batches %% cum %% 1: 137 51 51 2: 28 10 62 4: 18 6 69 8: 18 6 75 16: 4 1 77 32: 18 6 84 64: 5 1 86 128: 3 1 87 256: 34 12 100 mdc.lustre-MDT0001-mdc-ffff9c6ce7db4000.batch_stats= snapshot_time: 1790978283.708349473 secs.nsecs start_time: 1790978213.502852012 secs.nsecs elapsed_time: 70.205497461 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 1 seconds: 0.000000 unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9c6ce7db4000.statahead_batch_max=64 llite.lustre-ffff9c6ce7db4000.statahead_max=128