== sanity test 123ad: Verify batching statahead works correctly ========================================================== 21:03:41 (1789434221) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9023797c7000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9023797c7000.batch_stats=0 llite.lustre-ffff9023797c7000.statahead_max=32 llite.lustre-ffff9023797c7000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9023797c7000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9023797c7000.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 1.80 seconds: 55.41 ops/second llite.lustre-ffff9023797c7000.statahead_max=0 101 real 0m1.532s user 0m0.005s sys 0m0.286s 'ls -l' 100 files without statahead: 1 sec llite.lustre-ffff9023797c7000.statahead_max=32 llite.lustre-ffff9023797c7000.statahead_max=32 101 real 0m1.035s user 0m0.004s sys 0m0.147s 'ls -l' 100 files with statahead: 1 sec statahead total: 123 statahead wrong: 2 agl total: 118 list_total: 30 fname_total: 93 hit_total: 3989 miss_total: 1491 swrong=2 ewrong=2 - open/close 795 (time 1789434246.61 total 10.01 last 79.44) total: 900 open/close in 11.19 seconds: 80.45 ops/second llite.lustre-ffff9023797c7000.statahead_max=0 1001 real 0m17.902s user 0m0.037s sys 0m2.793s 'ls -l' 1000 files without statahead: 18 sec llite.lustre-ffff9023797c7000.statahead_max=32 llite.lustre-ffff9023797c7000.statahead_max=32 1001 real 0m7.865s user 0m0.025s sys 0m1.283s 'ls -l' 1000 files with statahead: 8 sec statahead total: 124 statahead wrong: 2 agl total: 119 list_total: 31 fname_total: 93 hit_total: 4985 miss_total: 1495 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 19 seconds llite.lustre-ffff9023797c7000.statahead_stats= statahead total: 124 statahead wrong: 2 agl total: 119 list_total: 31 fname_total: 93 hit_total: 4985 miss_total: 1495 mdc.lustre-MDT0000-mdc-ffff9023797c7000.batch_stats= snapshot_time: 1789434302.815214822 secs.nsecs start_time: 1789434225.904814492 secs.nsecs elapsed_time: 76.910400330 secs.nsecs subreqs per batch batches %% cum %% 1: 3 7 7 2: 1 2 10 4: 2 5 15 8: 2 5 21 16: 25 65 86 32: 5 13 100 mdc.lustre-MDT0001-mdc-ffff9023797c7000.batch_stats= snapshot_time: 1789434302.815304037 secs.nsecs start_time: 1789434225.904897595 secs.nsecs elapsed_time: 76.910406442 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 1 2 5 8: 4 10 15 16: 10 25 40 32: 24 60 100 - open/close 835 (time 1789434315.17 total 10.01 last 83.39) - open/close 1683 (time 1789434325.17 total 20.01 last 84.80) - open/close 2533 (time 1789434335.18 total 30.02 last 84.95) - open/close 3457 (time 1789434345.18 total 40.02 last 92.39) - open/close 4416 (time 1789434355.20 total 50.04 last 95.73) - open/close 5179 (time 1789434365.20 total 60.04 last 76.30) - open/close 6001 (time 1789434375.21 total 70.05 last 82.11) - open/close 6807 (time 1789434385.21 total 80.05 last 80.57) - open/close 7620 (time 1789434395.22 total 90.06 last 81.26) - open/close 8403 (time 1789434405.23 total 100.07 last 78.21) - open/close 9396 (time 1789434415.24 total 110.08 last 99.22) total: 10000 open/close in 117.25 seconds: 85.29 ops/second llite.lustre-ffff9023797c7000.statahead_batch_max=0 llite.lustre-ffff9023797c7000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9023797c7000.stats=clear mdc.lustre-MDT0001-mdc-ffff9023797c7000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m9.658s user 0m0.330s sys 0m14.568s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9023797c7000.statahead_batch_max=32 llite.lustre-ffff9023797c7000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9023797c7000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9023797c7000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9023797c7000.stats=clear mdc.lustre-MDT0001-mdc-ffff9023797c7000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m34.053s user 0m0.301s sys 0m13.240s batched statahead RPC= hit count=9986 mdc.lustre-MDT0000-mdc-ffff9023797c7000.batch_stats= snapshot_time: 1789434607.268041200 secs.nsecs start_time: 1789434500.608797187 secs.nsecs elapsed_time: 106.659244013 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9023797c7000.batch_stats= snapshot_time: 1789434607.268122180 secs.nsecs start_time: 1789434500.609008431 secs.nsecs elapsed_time: 106.659113749 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9023797c7000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9023797c7000.batch_stats=0 llite.lustre-ffff9023797c7000.statahead_max=2048 llite.lustre-ffff9023797c7000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9023797c7000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9023797c7000.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 1.79 seconds: 55.99 ops/second llite.lustre-ffff9023797c7000.statahead_max=0 101 real 0m2.955s user 0m0.006s sys 0m0.293s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9023797c7000.statahead_max=2048 llite.lustre-ffff9023797c7000.statahead_max=2048 101 real 0m0.928s user 0m0.007s sys 0m0.178s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10085 miss_total: 15 swrong=0 ewrong=0 - open/close 606 (time 1789434873.71 total 10.01 last 60.55) total: 900 open/close in 13.30 seconds: 67.65 ops/second llite.lustre-ffff9023797c7000.statahead_max=0 1001 real 0m21.880s user 0m0.030s sys 0m2.878s 'ls -l' 1000 files without statahead: 22 sec llite.lustre-ffff9023797c7000.statahead_max=2048 llite.lustre-ffff9023797c7000.statahead_max=2048 1001 real 0m8.030s user 0m0.034s sys 0m1.399s 'ls -l' 1000 files with statahead: 8 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11084 miss_total: 16 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 18 seconds llite.lustre-ffff9023797c7000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11084 miss_total: 16 mdc.lustre-MDT0000-mdc-ffff9023797c7000.batch_stats= snapshot_time: 1789434939.154768561 secs.nsecs start_time: 1789434609.492086306 secs.nsecs elapsed_time: 329.662682255 secs.nsecs subreqs per batch batches %% cum %% 1: 2 28 28 2: 0 0 28 4: 2 28 57 8: 1 14 71 16: 0 0 71 32: 0 0 71 64: 0 0 71 128: 0 0 71 256: 2 28 100 mdc.lustre-MDT0001-mdc-ffff9023797c7000.batch_stats= snapshot_time: 1789434939.155155626 secs.nsecs start_time: 1789434609.492289719 secs.nsecs elapsed_time: 329.662865907 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 11 11 8: 3 33 44 16: 1 11 55 32: 0 0 55 64: 2 22 77 128: 0 0 77 256: 2 22 100 - open/close 1029 (time 1789434951.49 total 10.02 last 102.69) - open/close 1937 (time 1789434961.50 total 20.02 last 90.78) - open/close 2911 (time 1789434971.50 total 30.03 last 97.34) - open/close 3941 (time 1789434981.51 total 40.03 last 102.95) - open/close 4871 (time 1789434991.51 total 50.04 last 92.94) - open/close 5848 (time 1789435001.51 total 60.04 last 97.68) - open/close 6814 (time 1789435011.51 total 70.04 last 96.60) - open/close 7700 (time 1789435021.52 total 80.05 last 88.56) - open/close 8534 (time 1789435031.53 total 90.05 last 83.34) - open/close 9443 (time 1789435041.53 total 100.06 last 90.85) total: 10000 open/close in 107.33 seconds: 93.17 ops/second llite.lustre-ffff9023797c7000.statahead_batch_max=0 llite.lustre-ffff9023797c7000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9023797c7000.stats=clear mdc.lustre-MDT0001-mdc-ffff9023797c7000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m1.374s user 0m0.319s sys 0m14.388s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9023797c7000.statahead_batch_max=256 llite.lustre-ffff9023797c7000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9023797c7000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9023797c7000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9023797c7000.stats=clear mdc.lustre-MDT0001-mdc-ffff9023797c7000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m40.604s user 0m0.255s sys 0m14.167s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9023797c7000.batch_stats= snapshot_time: 1789435169.747824504 secs.nsecs start_time: 1789435119.790300516 secs.nsecs elapsed_time: 49.957523988 secs.nsecs subreqs per batch batches %% cum %% 1: 191 51 51 2: 56 15 66 4: 34 9 75 8: 25 6 82 16: 8 2 84 32: 16 4 88 64: 7 1 90 128: 0 0 90 256: 34 9 100 mdc.lustre-MDT0001-mdc-ffff9023797c7000.batch_stats= snapshot_time: 1789435169.747888938 secs.nsecs start_time: 1789435119.790669302 secs.nsecs elapsed_time: 49.957219636 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9023797c7000.statahead_batch_max=64 llite.lustre-ffff9023797c7000.statahead_max=128