== sanity test 123ad: Verify batching statahead works correctly ========================================================== 19:38:13 (1789688293) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.batch_stats=0 llite.lustre-ffff9d32e6ff2000.statahead_max=32 llite.lustre-ffff9d32e6ff2000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.28 seconds: 43.78 ops/second llite.lustre-ffff9d32e6ff2000.statahead_max=0 101 real 0m1.639s user 0m0.003s sys 0m0.295s 'ls -l' 100 files without statahead: 1 sec llite.lustre-ffff9d32e6ff2000.statahead_max=32 llite.lustre-ffff9d32e6ff2000.statahead_max=32 101 real 0m1.005s user 0m0.004s sys 0m0.150s 'ls -l' 100 files with statahead: 1 sec statahead total: 91 statahead wrong: 1 agl total: 86 list_total: 29 fname_total: 61 hit_total: 3718 miss_total: 237 swrong=1 ewrong=1 total: 900 open/close in 9.82 seconds: 91.64 ops/second llite.lustre-ffff9d32e6ff2000.statahead_max=0 1001 real 0m20.955s user 0m0.036s sys 0m2.840s 'ls -l' 1000 files without statahead: 21 sec llite.lustre-ffff9d32e6ff2000.statahead_max=32 llite.lustre-ffff9d32e6ff2000.statahead_max=32 1001 real 0m7.189s user 0m0.019s sys 0m1.303s 'ls -l' 1000 files with statahead: 7 sec statahead total: 92 statahead wrong: 1 agl total: 87 list_total: 30 fname_total: 61 hit_total: 4716 miss_total: 239 swrong=1 ewrong=1 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 17 seconds llite.lustre-ffff9d32e6ff2000.statahead_stats= statahead total: 92 statahead wrong: 1 agl total: 87 list_total: 30 fname_total: 61 hit_total: 4716 miss_total: 239 mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.batch_stats= snapshot_time: 1789688374.381227205 secs.nsecs start_time: 1789688297.650552680 secs.nsecs elapsed_time: 76.730674525 secs.nsecs subreqs per batch batches %% cum %% 1: 2 5 5 2: 1 2 8 4: 2 5 13 8: 2 5 18 16: 27 72 91 32: 3 8 100 mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.batch_stats= snapshot_time: 1789688374.381290501 secs.nsecs start_time: 1789688297.650681292 secs.nsecs elapsed_time: 76.730609209 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 3 7 10 8: 1 2 12 16: 13 32 45 32: 22 55 100 - open/close 1041 (time 1789688386.86 total 10.00 last 104.06) - open/close 2130 (time 1789688396.87 total 20.01 last 108.80) - open/close 3128 (time 1789688406.88 total 30.02 last 99.69) - open/close 4137 (time 1789688416.89 total 40.03 last 100.85) - open/close 5115 (time 1789688426.89 total 50.03 last 97.80) - open/close 6076 (time 1789688436.89 total 60.03 last 96.10) - open/close 7011 (time 1789688446.89 total 70.04 last 93.45) - open/close 8041 (time 1789688456.90 total 80.04 last 102.94) - open/close 9031 (time 1789688466.90 total 90.04 last 98.98) - open/close 9974 (time 1789688476.92 total 100.06 last 94.16) total: 10000 open/close in 100.45 seconds: 99.55 ops/second llite.lustre-ffff9d32e6ff2000.statahead_batch_max=0 llite.lustre-ffff9d32e6ff2000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.stats=clear mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m48.623s user 0m0.277s sys 0m13.857s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9d32e6ff2000.statahead_batch_max=32 llite.lustre-ffff9d32e6ff2000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.stats=clear mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m28.748s user 0m0.285s sys 0m12.767s batched statahead RPC= hit count=9980 mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.batch_stats= snapshot_time: 1789688629.246868975 secs.nsecs start_time: 1789688532.180557496 secs.nsecs elapsed_time: 97.066311479 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 0 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.batch_stats= snapshot_time: 1789688629.246935867 secs.nsecs start_time: 1789688532.180697205 secs.nsecs elapsed_time: 97.066238662 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.batch_stats=0 llite.lustre-ffff9d32e6ff2000.statahead_max=2048 llite.lustre-ffff9d32e6ff2000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 1.87 seconds: 53.54 ops/second llite.lustre-ffff9d32e6ff2000.statahead_max=0 101 real 0m1.609s user 0m0.003s sys 0m0.280s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9d32e6ff2000.statahead_max=2048 llite.lustre-ffff9d32e6ff2000.statahead_max=2048 101 real 0m1.170s user 0m0.006s sys 0m0.184s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10079 miss_total: 21 swrong=0 ewrong=0 total: 900 open/close in 9.02 seconds: 99.83 ops/second llite.lustre-ffff9d32e6ff2000.statahead_max=0 1001 real 0m16.292s user 0m0.030s sys 0m2.773s 'ls -l' 1000 files without statahead: 17 sec llite.lustre-ffff9d32e6ff2000.statahead_max=2048 llite.lustre-ffff9d32e6ff2000.statahead_max=2048 1001 real 0m4.907s user 0m0.029s sys 0m1.188s 'ls -l' 1000 files with statahead: 5 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11078 miss_total: 22 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 14 seconds llite.lustre-ffff9d32e6ff2000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11078 miss_total: 22 mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.batch_stats= snapshot_time: 1789688912.778556760 secs.nsecs start_time: 1789688631.420093857 secs.nsecs elapsed_time: 281.358462903 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 3 60 60 16: 0 0 60 32: 0 0 60 64: 0 0 60 128: 0 0 60 256: 2 40 100 mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.batch_stats= snapshot_time: 1789688912.778624512 secs.nsecs start_time: 1789688631.420247421 secs.nsecs elapsed_time: 281.358377091 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 25 25 8: 1 12 37 16: 1 12 50 32: 0 0 50 64: 1 12 62 128: 1 12 75 256: 2 25 100 - open/close 985 (time 1789688925.61 total 10.00 last 98.48) - open/close 2068 (time 1789688935.61 total 20.00 last 108.29) - open/close 3155 (time 1789688945.62 total 30.01 last 108.60) - open/close 4193 (time 1789688955.62 total 40.01 last 103.77) - open/close 5269 (time 1789688965.63 total 50.02 last 107.49) - open/close 6290 (time 1789688975.63 total 60.03 last 102.08) - open/close 7411 (time 1789688985.64 total 70.03 last 112.08) - open/close 8511 (time 1789688995.64 total 80.03 last 109.98) - open/close 9694 (time 1789689005.64 total 90.03 last 118.30) total: 10000 open/close in 92.78 seconds: 107.78 ops/second llite.lustre-ffff9d32e6ff2000.statahead_batch_max=0 llite.lustre-ffff9d32e6ff2000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.stats=clear mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m50.169s user 0m0.277s sys 0m13.671s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9d32e6ff2000.statahead_batch_max=256 llite.lustre-ffff9d32e6ff2000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.stats=clear mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m55.247s user 0m0.252s sys 0m13.724s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9d32e6ff2000.batch_stats= snapshot_time: 1789689131.166899460 secs.nsecs start_time: 1789689066.226358182 secs.nsecs elapsed_time: 64.940541278 secs.nsecs subreqs per batch batches %% cum %% 1: 114 42 42 2: 42 15 57 4: 30 11 68 8: 18 6 75 16: 5 1 77 32: 18 6 84 64: 9 3 87 128: 0 0 87 256: 34 12 100 mdc.lustre-MDT0001-mdc-ffff9d32e6ff2000.batch_stats= snapshot_time: 1789689131.166970700 secs.nsecs start_time: 1789689066.226449514 secs.nsecs elapsed_time: 64.940521186 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9d32e6ff2000.statahead_batch_max=64 llite.lustre-ffff9d32e6ff2000.statahead_max=128