== sanity test 123ad: Verify batching statahead works correctly ========================================================== 06:32:51 (1787740371) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.batch_stats=0 llite.lustre-ffff8e8a810c4000.statahead_max=32 llite.lustre-ffff8e8a810c4000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 2.65 seconds: 37.72 ops/second llite.lustre-ffff8e8a810c4000.statahead_max=0 101 real 0m1.921s user 0m0.002s sys 0m0.316s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff8e8a810c4000.statahead_max=32 llite.lustre-ffff8e8a810c4000.statahead_max=32 101 real 0m1.216s user 0m0.004s sys 0m0.184s 'ls -l' 100 files with statahead: 1 sec statahead total: 115 statahead wrong: 2 agl total: 110 list_total: 29 fname_total: 86 hit_total: 4042 miss_total: 1264 swrong=2 ewrong=2 total: 900 open/close in 9.80 seconds: 91.80 ops/second llite.lustre-ffff8e8a810c4000.statahead_max=0 1001 real 0m21.224s user 0m0.040s sys 0m2.906s 'ls -l' 1000 files without statahead: 21 sec llite.lustre-ffff8e8a810c4000.statahead_max=32 llite.lustre-ffff8e8a810c4000.statahead_max=32 1001 real 0m12.236s user 0m0.025s sys 0m1.359s 'ls -l' 1000 files with statahead: 12 sec statahead total: 116 statahead wrong: 2 agl total: 111 list_total: 30 fname_total: 86 hit_total: 5037 miss_total: 1269 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 18 seconds llite.lustre-ffff8e8a810c4000.statahead_stats= statahead total: 116 statahead wrong: 2 agl total: 111 list_total: 30 fname_total: 86 hit_total: 5037 miss_total: 1269 mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.batch_stats= snapshot_time: 1787740463.784492683 secs.nsecs start_time: 1787740375.987358987 secs.nsecs elapsed_time: 87.797133696 secs.nsecs subreqs per batch batches %% cum %% 1: 2 5 5 2: 2 5 10 4: 2 5 16 8: 1 2 18 16: 25 67 86 32: 5 13 100 mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.batch_stats= snapshot_time: 1787740463.784559087 secs.nsecs start_time: 1787740375.991568876 secs.nsecs elapsed_time: 87.792990211 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 1 2 5 8: 4 10 15 16: 12 30 45 32: 22 55 100 - open/close 770 (time 1787740476.31 total 10.00 last 77.00) - open/close 1519 (time 1787740486.32 total 20.01 last 74.84) - open/close 2275 (time 1787740496.34 total 30.03 last 75.43) - open/close 3121 (time 1787740506.34 total 40.03 last 84.58) - open/close 3876 (time 1787740516.34 total 50.04 last 75.49) - open/close 4740 (time 1787740526.34 total 60.04 last 86.39) - open/close 5530 (time 1787740536.34 total 70.04 last 79.00) - open/close 6361 (time 1787740546.35 total 80.04 last 83.05) - open/close 7359 (time 1787740556.35 total 90.04 last 99.80) - open/close 8193 (time 1787740566.37 total 100.06 last 83.25) - open/close 8947 (time 1787740576.38 total 110.07 last 75.34) - open/close 9794 (time 1787740586.38 total 120.08 last 84.63) total: 10000 open/close in 122.92 seconds: 81.35 ops/second llite.lustre-ffff8e8a810c4000.statahead_batch_max=0 llite.lustre-ffff8e8a810c4000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.stats=clear mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m18.757s user 0m0.315s sys 0m14.794s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8e8a810c4000.statahead_batch_max=32 llite.lustre-ffff8e8a810c4000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.stats=clear mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m51.164s user 0m0.263s sys 0m13.483s batched statahead RPC= hit count=9980 mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.batch_stats= snapshot_time: 1787740798.052199617 secs.nsecs start_time: 1787740674.482347949 secs.nsecs elapsed_time: 123.569851668 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 0 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.batch_stats= snapshot_time: 1787740798.052268618 secs.nsecs start_time: 1787740674.482427312 secs.nsecs elapsed_time: 123.569841306 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.batch_stats=0 llite.lustre-ffff8e8a810c4000.statahead_max=2048 llite.lustre-ffff8e8a810c4000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.52 seconds: 39.66 ops/second llite.lustre-ffff8e8a810c4000.statahead_max=0 101 real 0m2.166s user 0m0.006s sys 0m0.300s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff8e8a810c4000.statahead_max=2048 llite.lustre-ffff8e8a810c4000.statahead_max=2048 101 real 0m0.637s user 0m0.003s sys 0m0.194s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10079 miss_total: 21 swrong=0 ewrong=0 - open/close 761 (time 1787741098.59 total 10.02 last 75.92) total: 900 open/close in 11.71 seconds: 76.86 ops/second llite.lustre-ffff8e8a810c4000.statahead_max=0 1001 real 0m24.711s user 0m0.034s sys 0m2.884s 'ls -l' 1000 files without statahead: 25 sec llite.lustre-ffff8e8a810c4000.statahead_max=2048 llite.lustre-ffff8e8a810c4000.statahead_max=2048 1001 real 0m5.139s user 0m0.029s sys 0m1.448s 'ls -l' 1000 files with statahead: 5 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11078 miss_total: 22 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 25 seconds llite.lustre-ffff8e8a810c4000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11078 miss_total: 22 mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.batch_stats= snapshot_time: 1787741169.257837043 secs.nsecs start_time: 1787740800.580607781 secs.nsecs elapsed_time: 368.677229262 secs.nsecs subreqs per batch batches %% cum %% 1: 2 33 33 2: 0 0 33 4: 2 33 66 8: 0 0 66 16: 0 0 66 32: 0 0 66 64: 0 0 66 128: 0 0 66 256: 2 33 100 mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.batch_stats= snapshot_time: 1787741169.257910850 secs.nsecs start_time: 1787740800.580804128 secs.nsecs elapsed_time: 368.677106722 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 11 11 8: 2 22 33 16: 2 22 55 32: 0 0 55 64: 2 22 77 128: 0 0 77 256: 2 22 100 - open/close 847 (time 1787741181.38 total 10.01 last 84.65) - open/close 1807 (time 1787741191.38 total 20.01 last 95.99) - open/close 2751 (time 1787741201.40 total 30.03 last 94.22) - open/close 3698 (time 1787741211.41 total 40.04 last 94.59) - open/close 4735 (time 1787741221.42 total 50.05 last 103.62) - open/close 5501 (time 1787741231.42 total 60.05 last 76.56) - open/close 6307 (time 1787741241.45 total 70.08 last 80.36) - open/close 6935 (time 1787741251.46 total 80.08 last 62.78) - open/close 7643 (time 1787741261.47 total 90.10 last 70.72) - open/close 8537 (time 1787741271.47 total 100.10 last 89.36) - open/close 9170 (time 1787741281.48 total 110.11 last 63.27) - open/close 9803 (time 1787741291.48 total 120.11 last 63.29) total: 10000 open/close in 122.22 seconds: 81.82 ops/second llite.lustre-ffff8e8a810c4000.statahead_batch_max=0 llite.lustre-ffff8e8a810c4000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.stats=clear mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m8.623s user 0m0.319s sys 0m14.298s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8e8a810c4000.statahead_batch_max=256 llite.lustre-ffff8e8a810c4000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.stats=clear mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m46.113s user 0m0.238s sys 0m15.676s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff8e8a810c4000.batch_stats= snapshot_time: 1787741428.272273487 secs.nsecs start_time: 1787741371.653435979 secs.nsecs elapsed_time: 56.618837508 secs.nsecs subreqs per batch batches %% cum %% 1: 107 42 42 2: 31 12 54 4: 35 13 68 8: 17 6 75 16: 8 3 78 32: 8 3 81 64: 11 4 86 128: 1 0 86 256: 34 13 100 mdc.lustre-MDT0001-mdc-ffff8e8a810c4000.batch_stats= snapshot_time: 1787741428.272352163 secs.nsecs start_time: 1787741371.653582087 secs.nsecs elapsed_time: 56.618770076 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff8e8a810c4000.statahead_batch_max=64 llite.lustre-ffff8e8a810c4000.statahead_max=128