== sanity test 123ad: Verify batching statahead works correctly ========================================================== 02:05:50 (1788847550) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9e959175e000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9e959175e000.batch_stats=0 llite.lustre-ffff9e959175e000.statahead_max=32 llite.lustre-ffff9e959175e000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9e959175e000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9e959175e000.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.16 seconds: 31.64 ops/second llite.lustre-ffff9e959175e000.statahead_max=0 101 real 0m2.049s user 0m0.009s sys 0m0.308s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9e959175e000.statahead_max=32 llite.lustre-ffff9e959175e000.statahead_max=32 101 real 0m1.168s user 0m0.005s sys 0m0.144s 'ls -l' 100 files with statahead: 2 sec statahead total: 135 statahead wrong: 2 agl total: 122 list_total: 31 fname_total: 102 hit_total: 3744 miss_total: 1608 swrong=2 ewrong=2 - open/close 747 (time 1788847580.30 total 10.00 last 74.68) total: 900 open/close in 12.67 seconds: 71.01 ops/second llite.lustre-ffff9e959175e000.statahead_max=0 1001 real 0m24.143s user 0m0.033s sys 0m3.068s 'ls -l' 1000 files without statahead: 24 sec llite.lustre-ffff9e959175e000.statahead_max=32 llite.lustre-ffff9e959175e000.statahead_max=32 1001 real 0m10.976s user 0m0.026s sys 0m1.381s 'ls -l' 1000 files with statahead: 11 sec statahead total: 136 statahead wrong: 2 agl total: 123 list_total: 32 fname_total: 102 hit_total: 4739 miss_total: 1613 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 21 seconds llite.lustre-ffff9e959175e000.statahead_stats= statahead total: 136 statahead wrong: 2 agl total: 123 list_total: 32 fname_total: 102 hit_total: 4739 miss_total: 1613 mdc.lustre-MDT0000-mdc-ffff9e959175e000.batch_stats= snapshot_time: 1788847655.181506431 secs.nsecs start_time: 1788847556.347153992 secs.nsecs elapsed_time: 98.834352439 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 2 5 5 4: 2 5 11 8: 2 5 16 16: 26 72 88 32: 4 11 100 mdc.lustre-MDT0001-mdc-ffff9e959175e000.batch_stats= snapshot_time: 1788847655.181577148 secs.nsecs start_time: 1788847556.348348432 secs.nsecs elapsed_time: 98.833228716 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 5 5 8: 2 5 10 16: 14 35 46 32: 21 53 100 - open/close 589 (time 1788847668.41 total 10.00 last 58.90) - open/close 1159 (time 1788847678.43 total 20.02 last 56.91) - open/close 1651 (time 1788847688.43 total 30.02 last 49.19) - open/close 2413 (time 1788847698.43 total 40.02 last 76.17) - open/close 3216 (time 1788847708.44 total 50.03 last 80.26) - open/close 3827 (time 1788847718.45 total 60.04 last 61.02) - open/close 4364 (time 1788847728.46 total 70.04 last 53.67) - open/close 5057 (time 1788847738.46 total 80.05 last 69.28) - open/close 5793 (time 1788847748.47 total 90.06 last 73.53) - open/close 6620 (time 1788847758.50 total 100.08 last 82.48) - open/close 7438 (time 1788847768.50 total 110.09 last 81.78) - open/close 8279 (time 1788847778.51 total 120.09 last 84.04) - open/close 9109 (time 1788847788.51 total 130.10 last 82.93) - open/close 9977 (time 1788847798.52 total 140.11 last 86.75) total: 10000 open/close in 140.35 seconds: 71.25 ops/second llite.lustre-ffff9e959175e000.statahead_batch_max=0 llite.lustre-ffff9e959175e000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9e959175e000.stats=clear mdc.lustre-MDT0001-mdc-ffff9e959175e000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m14.800s user 0m0.298s sys 0m14.273s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9e959175e000.statahead_batch_max=32 llite.lustre-ffff9e959175e000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9e959175e000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9e959175e000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9e959175e000.stats=clear mdc.lustre-MDT0001-mdc-ffff9e959175e000.stats=clear ls -l files in d123ad.sanity: 10001 real 2m8.910s user 0m0.329s sys 0m13.498s batched statahead RPC= hit count=9961 mdc.lustre-MDT0000-mdc-ffff9e959175e000.batch_stats= snapshot_time: 1788848026.291539109 secs.nsecs start_time: 1788847884.375099531 secs.nsecs elapsed_time: 141.916439578 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 314 99 100 mdc.lustre-MDT0001-mdc-ffff9e959175e000.batch_stats= snapshot_time: 1788848026.291602869 secs.nsecs start_time: 1788847884.375247264 secs.nsecs elapsed_time: 141.916355605 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9e959175e000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9e959175e000.batch_stats=0 llite.lustre-ffff9e959175e000.statahead_max=2048 llite.lustre-ffff9e959175e000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9e959175e000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9e959175e000.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 2.68 seconds: 37.38 ops/second llite.lustre-ffff9e959175e000.statahead_max=0 101 real 0m2.409s user 0m0.007s sys 0m0.308s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff9e959175e000.statahead_max=2048 llite.lustre-ffff9e959175e000.statahead_max=2048 101 real 0m0.942s user 0m0.004s sys 0m0.177s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10060 miss_total: 40 swrong=0 ewrong=0 - open/close 691 (time 1788848356.08 total 10.00 last 69.07) total: 900 open/close in 13.10 seconds: 68.72 ops/second llite.lustre-ffff9e959175e000.statahead_max=0 1001 real 0m23.628s user 0m0.043s sys 0m3.081s 'ls -l' 1000 files without statahead: 23 sec llite.lustre-ffff9e959175e000.statahead_max=2048 llite.lustre-ffff9e959175e000.statahead_max=2048 1001 real 0m5.876s user 0m0.019s sys 0m1.421s 'ls -l' 1000 files with statahead: 6 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11059 miss_total: 41 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 22 seconds llite.lustre-ffff9e959175e000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11059 miss_total: 41 mdc.lustre-MDT0000-mdc-ffff9e959175e000.batch_stats= snapshot_time: 1788848423.006671030 secs.nsecs start_time: 1788848030.142994847 secs.nsecs elapsed_time: 392.863676183 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 2 25 25 4: 1 12 37 8: 2 25 62 16: 0 0 62 32: 1 12 75 64: 0 0 75 128: 0 0 75 256: 2 25 100 mdc.lustre-MDT0001-mdc-ffff9e959175e000.batch_stats= snapshot_time: 1788848423.006774567 secs.nsecs start_time: 1788848030.143130377 secs.nsecs elapsed_time: 392.863644190 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 9 9 8: 2 18 27 16: 5 45 72 32: 0 0 72 64: 1 9 81 128: 0 0 81 256: 2 18 100 - open/close 763 (time 1788848436.17 total 10.00 last 76.27) - open/close 1674 (time 1788848446.17 total 20.01 last 91.02) - open/close 2519 (time 1788848456.18 total 30.02 last 84.48) - open/close 3227 (time 1788848466.18 total 40.02 last 70.76) - open/close 3925 (time 1788848476.18 total 50.02 last 69.79) - open/close 4576 (time 1788848486.20 total 60.03 last 65.01) - open/close 5257 (time 1788848496.20 total 70.04 last 68.05) - open/close 5880 (time 1788848506.23 total 80.07 last 62.14) - open/close 6506 (time 1788848516.24 total 90.08 last 62.55) - open/close 7162 (time 1788848526.24 total 100.08 last 65.56) - open/close 7815 (time 1788848536.25 total 110.09 last 65.24) - open/close 8408 (time 1788848546.26 total 120.10 last 59.24) - open/close 9112 (time 1788848556.27 total 130.11 last 70.37) - open/close 9867 (time 1788848566.28 total 140.12 last 75.41) total: 10000 open/close in 142.28 seconds: 70.28 ops/second llite.lustre-ffff9e959175e000.statahead_batch_max=0 llite.lustre-ffff9e959175e000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9e959175e000.stats=clear mdc.lustre-MDT0001-mdc-ffff9e959175e000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m3.512s user 0m0.313s sys 0m14.097s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9e959175e000.statahead_batch_max=256 llite.lustre-ffff9e959175e000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9e959175e000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9e959175e000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9e959175e000.stats=clear mdc.lustre-MDT0001-mdc-ffff9e959175e000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m55.539s user 0m0.326s sys 0m13.653s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9e959175e000.batch_stats= snapshot_time: 1788848708.556326793 secs.nsecs start_time: 1788848640.694805227 secs.nsecs elapsed_time: 67.861521566 secs.nsecs subreqs per batch batches %% cum %% 1: 96 41 41 2: 27 11 52 4: 27 11 64 8: 18 7 71 16: 7 2 74 32: 23 9 84 64: 0 0 84 128: 1 0 85 256: 35 14 100 mdc.lustre-MDT0001-mdc-ffff9e959175e000.batch_stats= snapshot_time: 1788848708.556390535 secs.nsecs start_time: 1788848640.694920687 secs.nsecs elapsed_time: 67.861469848 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9e959175e000.statahead_batch_max=64 llite.lustre-ffff9e959175e000.statahead_max=128