== sanity test 123ad: Verify batching statahead works correctly ========================================================== 15:28:03 (1783970883) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.batch_stats=0 llite.lustre-ffff8e83c8ad4800.statahead_max=32 llite.lustre-ffff8e83c8ad4800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 2.11 seconds: 47.46 ops/second llite.lustre-ffff8e83c8ad4800.statahead_max=0 101 real 0m2.443s user 0m0.007s sys 0m0.391s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff8e83c8ad4800.statahead_max=32 llite.lustre-ffff8e83c8ad4800.statahead_max=32 101 real 0m1.241s user 0m0.008s sys 0m0.224s 'ls -l' 100 files with statahead: 1 sec statahead total: 121 statahead wrong: 2 agl total: 116 list_total: 31 fname_total: 89 hit_total: 4201 miss_total: 1289 swrong=2 ewrong=2 - open/close 769 (time 1783970912.04 total 10.02 last 76.74) total: 900 open/close in 11.90 seconds: 75.66 ops/second llite.lustre-ffff8e83c8ad4800.statahead_max=0 1001 real 0m25.611s user 0m0.040s sys 0m3.577s 'ls -l' 1000 files without statahead: 26 sec llite.lustre-ffff8e83c8ad4800.statahead_max=32 llite.lustre-ffff8e83c8ad4800.statahead_max=32 1001 real 0m10.327s user 0m0.030s sys 0m1.626s 'ls -l' 1000 files with statahead: 11 sec statahead total: 122 statahead wrong: 2 agl total: 117 list_total: 32 fname_total: 89 hit_total: 5199 miss_total: 1291 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 20 seconds llite.lustre-ffff8e83c8ad4800.statahead_stats= statahead total: 122 statahead wrong: 2 agl total: 117 list_total: 32 fname_total: 89 hit_total: 5199 miss_total: 1291 mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.batch_stats= snapshot_time: 1783970984.309723763 secs.nsecs start_time: 1783970888.378763173 secs.nsecs elapsed_time: 95.930960590 secs.nsecs subreqs per batch batches %% cum %% 1: 2 5 5 2: 0 0 5 4: 3 8 13 8: 2 5 19 16: 24 66 86 32: 5 13 100 mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.batch_stats= snapshot_time: 1783970984.310164770 secs.nsecs start_time: 1783970888.378841076 secs.nsecs elapsed_time: 95.931323694 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 1 2 5 8: 4 10 15 16: 11 27 42 32: 23 57 100 - open/close 714 (time 1783970997.16 total 10.01 last 71.35) - open/close 1413 (time 1783971007.18 total 20.02 last 69.81) - open/close 2152 (time 1783971017.18 total 30.02 last 73.88) - open/close 2853 (time 1783971027.19 total 40.03 last 70.04) - open/close 3522 (time 1783971037.20 total 50.04 last 66.84) - open/close 4251 (time 1783971047.21 total 60.05 last 72.83) - open/close 4960 (time 1783971057.22 total 70.06 last 70.84) - open/close 5723 (time 1783971067.22 total 80.06 last 76.29) - open/close 6411 (time 1783971077.24 total 90.08 last 68.64) - open/close 7124 (time 1783971087.25 total 100.09 last 71.23) - open/close 7876 (time 1783971097.25 total 110.10 last 75.17) - open/close 8650 (time 1783971107.26 total 120.10 last 77.37) - open/close 9336 (time 1783971117.26 total 130.10 last 68.58) total: 10000 open/close in 138.91 seconds: 71.99 ops/second llite.lustre-ffff8e83c8ad4800.statahead_batch_max=0 llite.lustre-ffff8e83c8ad4800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.stats=clear mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m58.823s user 0m0.337s sys 0m19.316s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8e83c8ad4800.statahead_batch_max=32 llite.lustre-ffff8e83c8ad4800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.stats=clear mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m46.413s user 0m0.274s sys 0m16.294s batched statahead RPC= hit count=9984 mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.batch_stats= snapshot_time: 1783971317.874111305 secs.nsecs start_time: 1783971196.510250618 secs.nsecs elapsed_time: 121.363860687 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.batch_stats= snapshot_time: 1783971317.874174663 secs.nsecs start_time: 1783971196.510340194 secs.nsecs elapsed_time: 121.363834469 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.batch_stats=0 llite.lustre-ffff8e83c8ad4800.statahead_max=2048 llite.lustre-ffff8e83c8ad4800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.76 seconds: 36.17 ops/second llite.lustre-ffff8e83c8ad4800.statahead_max=0 101 real 0m2.432s user 0m0.005s sys 0m0.323s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff8e83c8ad4800.statahead_max=2048 llite.lustre-ffff8e83c8ad4800.statahead_max=2048 101 real 0m0.971s user 0m0.004s sys 0m0.194s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10083 miss_total: 17 swrong=0 ewrong=0 - open/close 715 (time 1783971654.47 total 10.02 last 71.36) total: 900 open/close in 12.81 seconds: 70.23 ops/second llite.lustre-ffff8e83c8ad4800.statahead_max=0 1001 real 0m24.872s user 0m0.036s sys 0m3.308s 'ls -l' 1000 files without statahead: 25 sec llite.lustre-ffff8e83c8ad4800.statahead_max=2048 llite.lustre-ffff8e83c8ad4800.statahead_max=2048 1001 real 0m8.138s user 0m0.036s sys 0m1.913s 'ls -l' 1000 files with statahead: 9 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11082 miss_total: 18 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 20 seconds llite.lustre-ffff8e83c8ad4800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11082 miss_total: 18 mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.batch_stats= snapshot_time: 1783971723.688248429 secs.nsecs start_time: 1783971320.512640931 secs.nsecs elapsed_time: 403.175607498 secs.nsecs subreqs per batch batches %% cum %% 1: 1 14 14 2: 0 0 14 4: 3 42 57 8: 1 14 71 16: 0 0 71 32: 0 0 71 64: 0 0 71 128: 0 0 71 256: 2 28 100 mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.batch_stats= snapshot_time: 1783971723.688306207 secs.nsecs start_time: 1783971320.521199825 secs.nsecs elapsed_time: 403.167106382 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 10 10 8: 3 30 40 16: 1 10 50 32: 2 20 70 64: 1 10 80 128: 0 0 80 256: 2 20 100 - open/close 699 (time 1783971736.01 total 10.00 last 69.89) - open/close 1403 (time 1783971746.02 total 20.01 last 70.34) - open/close 2139 (time 1783971756.03 total 30.03 last 73.47) - open/close 2811 (time 1783971766.04 total 40.03 last 67.15) - open/close 3466 (time 1783971776.05 total 50.04 last 65.45) - open/close 4174 (time 1783971786.05 total 60.05 last 70.76) - open/close 4852 (time 1783971796.06 total 70.05 last 67.75) - open/close 5548 (time 1783971806.08 total 80.07 last 69.49) - open/close 6250 (time 1783971816.08 total 90.07 last 70.19) - open/close 6971 (time 1783971826.08 total 100.08 last 72.06) - open/close 7731 (time 1783971836.09 total 110.08 last 75.94) - open/close 8501 (time 1783971846.09 total 120.09 last 76.99) - open/close 9256 (time 1783971856.10 total 130.09 last 75.47) total: 10000 open/close in 139.27 seconds: 71.81 ops/second llite.lustre-ffff8e83c8ad4800.statahead_batch_max=0 llite.lustre-ffff8e83c8ad4800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.stats=clear mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m57.075s user 0m0.320s sys 0m19.528s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8e83c8ad4800.statahead_batch_max=256 llite.lustre-ffff8e83c8ad4800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.stats=clear mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m52.014s user 0m0.317s sys 0m18.794s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff8e83c8ad4800.batch_stats= snapshot_time: 1783971994.914255282 secs.nsecs start_time: 1783971932.407237735 secs.nsecs elapsed_time: 62.507017547 secs.nsecs subreqs per batch batches %% cum %% 1: 317 58 58 2: 89 16 75 4: 45 8 83 8: 24 4 88 16: 7 1 89 32: 21 3 93 64: 2 0 93 128: 1 0 93 256: 33 6 100 mdc.lustre-MDT0001-mdc-ffff8e83c8ad4800.batch_stats= snapshot_time: 1783971994.914315684 secs.nsecs start_time: 1783971932.407320398 secs.nsecs elapsed_time: 62.506995286 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff8e83c8ad4800.statahead_batch_max=64 llite.lustre-ffff8e83c8ad4800.statahead_max=128