== sanity test 123ad: Verify batching statahead works correctly ========================================================== 13:36:37 (1790789797) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.batch_stats=0 llite.lustre-ffff9f7242d0d000.statahead_max=32 llite.lustre-ffff9f7242d0d000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.batch_stats=0 striped dir -i1 -c2 -H all_char /mnt/lustre/d123ad.sanity total: 100 open/close in 4.00 seconds: 24.98 ops/second llite.lustre-ffff9f7242d0d000.statahead_max=0 101 real 0m2.622s user 0m0.004s sys 0m0.328s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9f7242d0d000.statahead_max=32 llite.lustre-ffff9f7242d0d000.statahead_max=32 101 real 0m0.856s user 0m0.006s sys 0m0.171s 'ls -l' 100 files with statahead: 1 sec statahead total: 118 statahead wrong: 2 agl total: 113 list_total: 31 fname_total: 75 hit_total: 4175 miss_total: 1283 swrong=2 ewrong=2 - open/close 687 (time 1790789829.28 total 10.01 last 68.65) total: 900 open/close in 13.16 seconds: 68.37 ops/second llite.lustre-ffff9f7242d0d000.statahead_max=0 1001 real 0m21.432s user 0m0.030s sys 0m2.818s 'ls -l' 1000 files without statahead: 21 sec llite.lustre-ffff9f7242d0d000.statahead_max=32 llite.lustre-ffff9f7242d0d000.statahead_max=32 1001 real 0m9.804s user 0m0.031s sys 0m1.342s 'ls -l' 1000 files with statahead: 9 sec statahead total: 119 statahead wrong: 2 agl total: 114 list_total: 32 fname_total: 75 hit_total: 5169 miss_total: 1289 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 26 seconds llite.lustre-ffff9f7242d0d000.statahead_stats= statahead total: 119 statahead wrong: 2 agl total: 114 list_total: 32 fname_total: 75 hit_total: 5169 miss_total: 1289 mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.batch_stats= snapshot_time: 1790789901.789721469 secs.nsecs start_time: 1790789805.015850190 secs.nsecs elapsed_time: 96.773871279 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 3 7 7 8: 3 7 15 16: 17 43 58 32: 16 41 100 mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.batch_stats= snapshot_time: 1790789901.789784884 secs.nsecs start_time: 1790789805.016277133 secs.nsecs elapsed_time: 96.773507751 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 5 5 8: 4 10 15 16: 18 46 61 32: 15 38 100 - open/close 934 (time 1790789914.28 total 10.00 last 93.38) - open/close 1826 (time 1790789924.29 total 20.01 last 89.17) - open/close 2698 (time 1790789934.29 total 30.01 last 87.18) - open/close 3615 (time 1790789944.31 total 40.03 last 91.53) - open/close 4427 (time 1790789954.32 total 50.04 last 81.11) - open/close 5154 (time 1790789964.33 total 60.05 last 72.58) - open/close 5871 (time 1790789974.34 total 70.06 last 71.68) - open/close 6543 (time 1790789984.34 total 80.06 last 67.16) - open/close 7098 (time 1790789994.35 total 90.07 last 55.47) - open/close 7770 (time 1790790004.35 total 100.07 last 67.19) - open/close 8540 (time 1790790014.35 total 110.07 last 76.96) - open/close 9327 (time 1790790024.36 total 120.08 last 78.67) total: 10000 open/close in 129.47 seconds: 77.24 ops/second llite.lustre-ffff9f7242d0d000.statahead_batch_max=0 llite.lustre-ffff9f7242d0d000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.stats=clear mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m0.625s user 0m0.267s sys 0m15.458s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9f7242d0d000.statahead_batch_max=32 llite.lustre-ffff9f7242d0d000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.stats=clear mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m31.256s user 0m0.243s sys 0m13.957s batched statahead RPC= hit count=9984 mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.batch_stats= snapshot_time: 1790790204.569150980 secs.nsecs start_time: 1790790104.239275996 secs.nsecs elapsed_time: 100.329874984 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.batch_stats= snapshot_time: 1790790204.569230871 secs.nsecs start_time: 1790790104.239416619 secs.nsecs elapsed_time: 100.329814252 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.batch_stats=0 llite.lustre-ffff9f7242d0d000.statahead_max=2048 llite.lustre-ffff9f7242d0d000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 2.70 seconds: 37.04 ops/second llite.lustre-ffff9f7242d0d000.statahead_max=0 101 real 0m2.166s user 0m0.006s sys 0m0.309s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9f7242d0d000.statahead_max=2048 llite.lustre-ffff9f7242d0d000.statahead_max=2048 101 real 0m0.793s user 0m0.004s sys 0m0.173s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10083 miss_total: 17 swrong=0 ewrong=0 - open/close 847 (time 1790790518.98 total 10.00 last 84.67) total: 900 open/close in 10.65 seconds: 84.49 ops/second llite.lustre-ffff9f7242d0d000.statahead_max=0 1001 real 0m20.550s user 0m0.037s sys 0m2.774s 'ls -l' 1000 files without statahead: 20 sec llite.lustre-ffff9f7242d0d000.statahead_max=2048 llite.lustre-ffff9f7242d0d000.statahead_max=2048 1001 real 0m6.463s user 0m0.026s sys 0m1.377s 'ls -l' 1000 files with statahead: 6 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11082 miss_total: 18 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 21 seconds llite.lustre-ffff9f7242d0d000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11082 miss_total: 18 mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.batch_stats= snapshot_time: 1790790579.069974072 secs.nsecs start_time: 1790790207.028463394 secs.nsecs elapsed_time: 372.041510678 secs.nsecs subreqs per batch batches %% cum %% 1: 2 33 33 2: 1 16 50 4: 1 16 66 8: 0 0 66 16: 0 0 66 32: 0 0 66 64: 0 0 66 128: 0 0 66 256: 2 33 100 mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.batch_stats= snapshot_time: 1790790579.070343654 secs.nsecs start_time: 1790790207.028812992 secs.nsecs elapsed_time: 372.041530662 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 3 33 33 16: 2 22 55 32: 1 11 66 64: 1 11 77 128: 0 0 77 256: 2 22 100 - open/close 813 (time 1790790591.60 total 10.02 last 81.16) - open/close 1521 (time 1790790601.61 total 20.03 last 70.73) - open/close 2136 (time 1790790611.61 total 30.03 last 61.47) - open/close 2840 (time 1790790621.62 total 40.04 last 70.32) - open/close 3697 (time 1790790631.63 total 50.05 last 85.66) - open/close 4534 (time 1790790641.64 total 60.06 last 83.62) - open/close 5279 (time 1790790651.69 total 70.11 last 74.12) - open/close 5895 (time 1790790661.70 total 80.13 last 61.49) - open/close 6513 (time 1790790671.72 total 90.14 last 61.68) - open/close 7145 (time 1790790681.73 total 100.15 last 63.16) - open/close 7853 (time 1790790691.74 total 110.16 last 70.75) - open/close 8709 (time 1790790701.74 total 120.16 last 85.58) - open/close 9447 (time 1790790711.74 total 130.16 last 73.77) total: 10000 open/close in 138.91 seconds: 71.99 ops/second llite.lustre-ffff9f7242d0d000.statahead_batch_max=0 llite.lustre-ffff9f7242d0d000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.stats=clear mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m0.531s user 0m0.285s sys 0m15.169s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9f7242d0d000.statahead_batch_max=256 llite.lustre-ffff9f7242d0d000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.stats=clear mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m49.692s user 0m0.257s sys 0m14.950s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9f7242d0d000.batch_stats= snapshot_time: 1790790849.755717048 secs.nsecs start_time: 1790790790.988205644 secs.nsecs elapsed_time: 58.767511404 secs.nsecs subreqs per batch batches %% cum %% 1: 146 45 45 2: 50 15 61 4: 35 10 71 8: 25 7 79 16: 11 3 83 32: 12 3 86 64: 6 1 88 128: 2 0 89 256: 34 10 100 mdc.lustre-MDT0001-mdc-ffff9f7242d0d000.batch_stats= snapshot_time: 1790790849.755786612 secs.nsecs start_time: 1790790790.988286171 secs.nsecs elapsed_time: 58.767500441 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9f7242d0d000.statahead_batch_max=64 llite.lustre-ffff9f7242d0d000.statahead_max=128