== sanity test 123ad: Verify batching statahead works correctly ========================================================== 00:58:58 (1788843538) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.batch_stats=0 llite.lustre-ffff9d1518b2c800.statahead_max=32 llite.lustre-ffff9d1518b2c800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.67 seconds: 37.44 ops/second llite.lustre-ffff9d1518b2c800.statahead_max=0 101 real 0m2.768s user 0m0.005s sys 0m0.340s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9d1518b2c800.statahead_max=32 llite.lustre-ffff9d1518b2c800.statahead_max=32 101 real 0m1.044s user 0m0.007s sys 0m0.161s 'ls -l' 100 files with statahead: 1 sec statahead total: 96 statahead wrong: 2 agl total: 91 list_total: 31 fname_total: 65 hit_total: 4307 miss_total: 1283 swrong=2 ewrong=2 - open/close 728 (time 1788843566.36 total 10.01 last 72.71) total: 900 open/close in 12.38 seconds: 72.72 ops/second llite.lustre-ffff9d1518b2c800.statahead_max=0 1001 real 0m24.754s user 0m0.045s sys 0m2.965s 'ls -l' 1000 files without statahead: 25 sec llite.lustre-ffff9d1518b2c800.statahead_max=32 llite.lustre-ffff9d1518b2c800.statahead_max=32 1001 real 0m10.053s user 0m0.033s sys 0m1.435s 'ls -l' 1000 files with statahead: 10 sec statahead total: 97 statahead wrong: 2 agl total: 92 list_total: 32 fname_total: 65 hit_total: 5305 miss_total: 1285 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 22 seconds llite.lustre-ffff9d1518b2c800.statahead_stats= statahead total: 97 statahead wrong: 2 agl total: 92 list_total: 32 fname_total: 65 hit_total: 5305 miss_total: 1285 mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.batch_stats= snapshot_time: 1788843638.656047956 secs.nsecs start_time: 1788843543.114397779 secs.nsecs elapsed_time: 95.541650177 secs.nsecs subreqs per batch batches %% cum %% 1: 2 5 5 2: 0 0 5 4: 3 7 12 8: 0 0 12 16: 20 51 64 32: 14 35 100 mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.batch_stats= snapshot_time: 1788843638.656156133 secs.nsecs start_time: 1788843543.115294244 secs.nsecs elapsed_time: 95.540861889 secs.nsecs subreqs per batch batches %% cum %% 1: 2 4 4 2: 0 0 4 4: 1 2 7 8: 6 14 21 16: 20 48 70 32: 12 29 100 - open/close 726 (time 1788843650.92 total 10.01 last 72.55) - open/close 1495 (time 1788843660.92 total 20.01 last 76.87) - open/close 2267 (time 1788843670.92 total 30.02 last 77.16) - open/close 3044 (time 1788843680.94 total 40.03 last 77.59) - open/close 3736 (time 1788843690.94 total 50.03 last 69.17) - open/close 4470 (time 1788843700.96 total 60.05 last 73.31) - open/close 5156 (time 1788843710.96 total 70.05 last 68.58) - open/close 5911 (time 1788843720.97 total 80.06 last 75.45) - open/close 6743 (time 1788843730.98 total 90.07 last 83.11) - open/close 7520 (time 1788843740.98 total 100.07 last 77.70) - open/close 8364 (time 1788843750.98 total 110.07 last 84.35) - open/close 9101 (time 1788843760.99 total 120.08 last 73.67) - open/close 9870 (time 1788843771.00 total 130.09 last 76.81) total: 10000 open/close in 132.14 seconds: 75.68 ops/second llite.lustre-ffff9d1518b2c800.statahead_batch_max=0 llite.lustre-ffff9d1518b2c800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.stats=clear mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m52.448s user 0m0.318s sys 0m16.952s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9d1518b2c800.statahead_batch_max=32 llite.lustre-ffff9d1518b2c800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.stats=clear mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m26.536s user 0m0.277s sys 0m14.106s batched statahead RPC= hit count=9987 mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.batch_stats= snapshot_time: 1788843930.130937903 secs.nsecs start_time: 1788843835.426496045 secs.nsecs elapsed_time: 94.704441858 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.batch_stats= snapshot_time: 1788843930.130999544 secs.nsecs start_time: 1788843835.426625845 secs.nsecs elapsed_time: 94.704373699 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.batch_stats=0 llite.lustre-ffff9d1518b2c800.statahead_max=2048 llite.lustre-ffff9d1518b2c800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 2.36 seconds: 42.32 ops/second llite.lustre-ffff9d1518b2c800.statahead_max=0 101 real 0m2.251s user 0m0.007s sys 0m0.325s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9d1518b2c800.statahead_max=2048 llite.lustre-ffff9d1518b2c800.statahead_max=2048 101 real 0m0.748s user 0m0.004s sys 0m0.194s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10086 miss_total: 14 swrong=0 ewrong=0 - open/close 835 (time 1788844223.42 total 10.00 last 83.47) total: 900 open/close in 10.83 seconds: 83.09 ops/second llite.lustre-ffff9d1518b2c800.statahead_max=0 1001 real 0m21.044s user 0m0.036s sys 0m3.025s 'ls -l' 1000 files without statahead: 21 sec llite.lustre-ffff9d1518b2c800.statahead_max=2048 llite.lustre-ffff9d1518b2c800.statahead_max=2048 1001 real 0m6.141s user 0m0.022s sys 0m1.337s 'ls -l' 1000 files with statahead: 6 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11085 miss_total: 15 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 23 seconds llite.lustre-ffff9d1518b2c800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11085 miss_total: 15 mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.batch_stats= snapshot_time: 1788844285.844880241 secs.nsecs start_time: 1788843932.610604931 secs.nsecs elapsed_time: 353.234275310 secs.nsecs subreqs per batch batches %% cum %% 1: 1 16 16 2: 1 16 33 4: 2 33 66 8: 0 0 66 16: 0 0 66 32: 0 0 66 64: 0 0 66 128: 0 0 66 256: 2 33 100 mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.batch_stats= snapshot_time: 1788844285.844948341 secs.nsecs start_time: 1788843932.610834481 secs.nsecs elapsed_time: 353.234113860 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 12 12 8: 2 25 37 16: 1 12 50 32: 0 0 50 64: 1 12 62 128: 1 12 75 256: 2 25 100 - open/close 768 (time 1788844298.34 total 10.01 last 76.75) - open/close 1483 (time 1788844308.34 total 20.01 last 71.49) - open/close 2176 (time 1788844318.35 total 30.02 last 69.20) - open/close 2863 (time 1788844328.37 total 40.04 last 68.55) - open/close 3626 (time 1788844338.38 total 50.05 last 76.25) - open/close 4424 (time 1788844348.39 total 60.06 last 79.72) - open/close 5264 (time 1788844358.40 total 70.07 last 83.94) - open/close 6057 (time 1788844368.42 total 80.09 last 79.11) - open/close 6773 (time 1788844378.42 total 90.09 last 71.58) - open/close 7644 (time 1788844388.44 total 100.11 last 86.95) - open/close 8497 (time 1788844398.45 total 110.12 last 85.24) - open/close 9345 (time 1788844408.45 total 120.12 last 84.77) total: 10000 open/close in 128.27 seconds: 77.96 ops/second llite.lustre-ffff9d1518b2c800.statahead_batch_max=0 llite.lustre-ffff9d1518b2c800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.stats=clear mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m58.411s user 0m0.314s sys 0m15.965s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9d1518b2c800.statahead_batch_max=256 llite.lustre-ffff9d1518b2c800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.stats=clear mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m47.597s user 0m0.289s sys 0m15.846s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9d1518b2c800.batch_stats= snapshot_time: 1788844541.251607058 secs.nsecs start_time: 1788844483.915630575 secs.nsecs elapsed_time: 57.335976483 secs.nsecs subreqs per batch batches %% cum %% 1: 110 44 44 2: 41 16 61 4: 24 9 70 8: 6 2 73 16: 7 2 76 32: 12 4 80 64: 12 4 85 128: 1 0 86 256: 34 13 100 mdc.lustre-MDT0001-mdc-ffff9d1518b2c800.batch_stats= snapshot_time: 1788844541.251666934 secs.nsecs start_time: 1788844483.915839915 secs.nsecs elapsed_time: 57.335827019 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9d1518b2c800.statahead_batch_max=64 llite.lustre-ffff9d1518b2c800.statahead_max=128