== sanity test 123ad: Verify batching statahead works correctly ========================================================== 14:26:06 (1788546366) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.batch_stats=0 llite.lustre-ffff9eefff2fe000.statahead_max=32 llite.lustre-ffff9eefff2fe000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.batch_stats=0 striped dir -i1 -c2 -H all_char /mnt/lustre/d123ad.sanity total: 100 open/close in 3.32 seconds: 30.09 ops/second llite.lustre-ffff9eefff2fe000.statahead_max=0 101 real 0m3.760s user 0m0.008s sys 0m0.411s 'ls -l' 100 files without statahead: 4 sec llite.lustre-ffff9eefff2fe000.statahead_max=32 llite.lustre-ffff9eefff2fe000.statahead_max=32 101 real 0m1.305s user 0m0.005s sys 0m0.176s 'ls -l' 100 files with statahead: 1 sec statahead total: 129 statahead wrong: 2 agl total: 123 list_total: 32 fname_total: 82 hit_total: 4057 miss_total: 1571 swrong=2 ewrong=2 - open/close 631 (time 1788546401.46 total 10.01 last 63.05) total: 900 open/close in 14.48 seconds: 62.17 ops/second llite.lustre-ffff9eefff2fe000.statahead_max=0 1001 real 0m23.593s user 0m0.040s sys 0m3.092s 'ls -l' 1000 files without statahead: 24 sec llite.lustre-ffff9eefff2fe000.statahead_max=32 llite.lustre-ffff9eefff2fe000.statahead_max=32 1001 real 0m8.596s user 0m0.027s sys 0m1.400s 'ls -l' 1000 files with statahead: 8 sec statahead total: 130 statahead wrong: 2 agl total: 124 list_total: 33 fname_total: 82 hit_total: 5056 miss_total: 1572 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 21 seconds llite.lustre-ffff9eefff2fe000.statahead_stats= statahead total: 130 statahead wrong: 2 agl total: 124 list_total: 33 fname_total: 82 hit_total: 5056 miss_total: 1572 mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.batch_stats= snapshot_time: 1788546471.786401302 secs.nsecs start_time: 1788546373.521565181 secs.nsecs elapsed_time: 98.264836121 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 3 7 10 8: 2 5 15 16: 17 43 58 32: 16 41 100 mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.batch_stats= snapshot_time: 1788546471.786574045 secs.nsecs start_time: 1788546373.521660006 secs.nsecs elapsed_time: 98.264914039 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 1 2 5 4: 2 5 10 8: 4 10 20 16: 18 45 65 32: 14 35 100 - open/close 668 (time 1788546485.02 total 10.01 last 66.76) - open/close 1358 (time 1788546495.04 total 20.02 last 68.90) - open/close 2017 (time 1788546505.04 total 30.03 last 65.85) - open/close 2705 (time 1788546515.06 total 40.04 last 68.70) - open/close 3347 (time 1788546525.08 total 50.06 last 64.06) - open/close 4030 (time 1788546535.10 total 60.08 last 68.17) - open/close 4705 (time 1788546545.11 total 70.09 last 67.44) - open/close 5377 (time 1788546555.12 total 80.10 last 67.12) - open/close 6062 (time 1788546565.13 total 90.11 last 68.45) - open/close 6731 (time 1788546575.15 total 100.14 last 66.72) - open/close 7424 (time 1788546585.16 total 110.15 last 69.25) - open/close 8093 (time 1788546595.17 total 120.15 last 66.85) - open/close 8722 (time 1788546605.17 total 130.16 last 62.87) - open/close 9376 (time 1788546615.19 total 140.17 last 65.29) - open/close 9964 (time 1788546625.19 total 150.18 last 58.78) total: 10000 open/close in 150.93 seconds: 66.26 ops/second llite.lustre-ffff9eefff2fe000.statahead_batch_max=0 llite.lustre-ffff9eefff2fe000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.stats=clear mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m5.446s user 0m0.353s sys 0m16.445s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9eefff2fe000.statahead_batch_max=32 llite.lustre-ffff9eefff2fe000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.stats=clear mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m46.564s user 0m0.338s sys 0m15.206s batched statahead RPC= hit count=9988 mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.batch_stats= snapshot_time: 1788546818.129743570 secs.nsecs start_time: 1788546701.749179419 secs.nsecs elapsed_time: 116.380564151 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.batch_stats= snapshot_time: 1788546818.129826010 secs.nsecs start_time: 1788546701.749315074 secs.nsecs elapsed_time: 116.380510936 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.batch_stats=0 llite.lustre-ffff9eefff2fe000.statahead_max=2048 llite.lustre-ffff9eefff2fe000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.batch_stats=0 striped dir -i1 -c2 -H all_char /mnt/lustre/d123ad.sanity total: 100 open/close in 3.19 seconds: 31.39 ops/second llite.lustre-ffff9eefff2fe000.statahead_max=0 101 real 0m2.551s user 0m0.005s sys 0m0.367s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff9eefff2fe000.statahead_max=2048 llite.lustre-ffff9eefff2fe000.statahead_max=2048 101 real 0m0.841s user 0m0.004s sys 0m0.201s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10087 miss_total: 13 swrong=0 ewrong=0 - open/close 660 (time 1788547158.28 total 10.00 last 65.99) total: 900 open/close in 13.62 seconds: 66.09 ops/second llite.lustre-ffff9eefff2fe000.statahead_max=0 1001 real 0m25.134s user 0m0.042s sys 0m3.087s 'ls -l' 1000 files without statahead: 26 sec llite.lustre-ffff9eefff2fe000.statahead_max=2048 llite.lustre-ffff9eefff2fe000.statahead_max=2048 1001 real 0m4.698s user 0m0.028s sys 0m1.707s 'ls -l' 1000 files with statahead: 4 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11086 miss_total: 14 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 24 seconds llite.lustre-ffff9eefff2fe000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11086 miss_total: 14 mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.batch_stats= snapshot_time: 1788547228.300655963 secs.nsecs start_time: 1788546820.526554782 secs.nsecs elapsed_time: 407.774101181 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 3 33 33 8: 1 11 44 16: 2 22 66 32: 1 11 77 64: 0 0 77 128: 0 0 77 256: 2 22 100 mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.batch_stats= snapshot_time: 1788547228.300734032 secs.nsecs start_time: 1788546820.528874520 secs.nsecs elapsed_time: 407.771859512 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 22 22 8: 4 44 66 16: 0 0 66 32: 0 0 66 64: 1 11 77 128: 0 0 77 256: 2 22 100 - open/close 668 (time 1788547241.78 total 10.01 last 66.74) - open/close 1360 (time 1788547251.78 total 20.02 last 69.15) - open/close 1939 (time 1788547261.79 total 30.03 last 57.85) - open/close 2527 (time 1788547271.80 total 40.03 last 58.78) - open/close 3185 (time 1788547281.80 total 50.03 last 65.80) - open/close 3835 (time 1788547291.81 total 60.04 last 64.92) - open/close 4477 (time 1788547301.81 total 70.05 last 64.15) - open/close 5155 (time 1788547311.82 total 80.06 last 67.75) - open/close 5828 (time 1788547321.83 total 90.06 last 67.26) - open/close 6446 (time 1788547331.83 total 100.06 last 61.78) - open/close 7096 (time 1788547341.83 total 110.07 last 64.98) - open/close 7784 (time 1788547351.83 total 120.07 last 68.79) - open/close 8382 (time 1788547361.86 total 130.09 last 59.66) - open/close 9025 (time 1788547371.86 total 140.09 last 64.30) - open/close 9703 (time 1788547381.87 total 150.10 last 67.73) total: 10000 open/close in 155.16 seconds: 64.45 ops/second llite.lustre-ffff9eefff2fe000.statahead_batch_max=0 llite.lustre-ffff9eefff2fe000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.stats=clear mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m18.060s user 0m0.337s sys 0m15.570s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9eefff2fe000.statahead_batch_max=256 llite.lustre-ffff9eefff2fe000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.stats=clear mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m1.040s user 0m0.359s sys 0m15.346s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9eefff2fe000.batch_stats= snapshot_time: 1788547547.618769714 secs.nsecs start_time: 1788547475.038890066 secs.nsecs elapsed_time: 72.579879648 secs.nsecs subreqs per batch batches %% cum %% 1: 151 52 52 2: 30 10 63 4: 23 8 71 8: 14 4 76 16: 7 2 78 32: 21 7 86 64: 5 1 87 128: 0 0 87 256: 35 12 100 mdc.lustre-MDT0001-mdc-ffff9eefff2fe000.batch_stats= snapshot_time: 1788547547.619196298 secs.nsecs start_time: 1788547475.039923039 secs.nsecs elapsed_time: 72.579273259 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9eefff2fe000.statahead_batch_max=64 llite.lustre-ffff9eefff2fe000.statahead_max=128