== sanity test 123ad: Verify batching statahead works correctly ========================================================== 15:43:27 (1787859807) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff913d02dae000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff913d02dae000.batch_stats=0 llite.lustre-ffff913d02dae000.statahead_max=32 llite.lustre-ffff913d02dae000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff913d02dae000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff913d02dae000.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.97 seconds: 33.69 ops/second llite.lustre-ffff913d02dae000.statahead_max=0 101 real 0m2.608s user 0m0.008s sys 0m0.350s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff913d02dae000.statahead_max=32 llite.lustre-ffff913d02dae000.statahead_max=32 101 real 0m1.090s user 0m0.003s sys 0m0.175s 'ls -l' 100 files with statahead: 1 sec statahead total: 156 statahead wrong: 2 agl total: 149 list_total: 30 fname_total: 112 hit_total: 4142 miss_total: 772 swrong=2 ewrong=2 - open/close 709 (time 1787859834.73 total 10.00 last 70.87) total: 900 open/close in 12.68 seconds: 70.96 ops/second llite.lustre-ffff913d02dae000.statahead_max=0 1001 real 0m23.259s user 0m0.035s sys 0m3.128s 'ls -l' 1000 files without statahead: 24 sec llite.lustre-ffff913d02dae000.statahead_max=32 llite.lustre-ffff913d02dae000.statahead_max=32 1001 real 0m8.645s user 0m0.022s sys 0m1.380s 'ls -l' 1000 files with statahead: 9 sec statahead total: 157 statahead wrong: 2 agl total: 150 list_total: 31 fname_total: 112 hit_total: 5137 miss_total: 777 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 23 seconds llite.lustre-ffff913d02dae000.statahead_stats= statahead total: 157 statahead wrong: 2 agl total: 150 list_total: 31 fname_total: 112 hit_total: 5137 miss_total: 777 mdc.lustre-MDT0000-mdc-ffff913d02dae000.batch_stats= snapshot_time: 1787859904.460001168 secs.nsecs start_time: 1787859811.809838974 secs.nsecs elapsed_time: 92.650162194 secs.nsecs subreqs per batch batches %% cum %% 1: 4 10 10 2: 2 5 15 4: 2 5 20 8: 1 2 23 16: 23 58 82 32: 7 17 100 mdc.lustre-MDT0001-mdc-ffff913d02dae000.batch_stats= snapshot_time: 1787859904.460081944 secs.nsecs start_time: 1787859811.809982942 secs.nsecs elapsed_time: 92.650099002 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 2 5 7 8: 2 5 12 16: 9 22 35 32: 26 65 100 - open/close 715 (time 1787859917.52 total 10.01 last 71.46) - open/close 1417 (time 1787859927.53 total 20.02 last 70.13) - open/close 2132 (time 1787859937.54 total 30.03 last 71.42) - open/close 2827 (time 1787859947.54 total 40.03 last 69.48) - open/close 3490 (time 1787859957.55 total 50.04 last 66.26) - open/close 4241 (time 1787859967.55 total 60.04 last 75.05) - open/close 4985 (time 1787859977.56 total 70.04 last 74.40) - open/close 5612 (time 1787859987.56 total 80.04 last 62.69) - open/close 6194 (time 1787859997.56 total 90.05 last 58.19) - open/close 6990 (time 1787860007.56 total 100.05 last 79.59) - open/close 7733 (time 1787860017.57 total 110.06 last 74.23) - open/close 8428 (time 1787860027.58 total 120.06 last 69.46) - open/close 9184 (time 1787860037.58 total 130.07 last 75.55) - open/close 9990 (time 1787860047.58 total 140.07 last 80.59) total: 10000 open/close in 140.19 seconds: 71.33 ops/second llite.lustre-ffff913d02dae000.statahead_batch_max=0 llite.lustre-ffff913d02dae000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff913d02dae000.stats=clear mdc.lustre-MDT0001-mdc-ffff913d02dae000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m1.641s user 0m0.378s sys 0m17.189s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff913d02dae000.statahead_batch_max=32 llite.lustre-ffff913d02dae000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff913d02dae000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff913d02dae000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff913d02dae000.stats=clear mdc.lustre-MDT0001-mdc-ffff913d02dae000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m38.346s user 0m0.293s sys 0m14.105s batched statahead RPC= hit count=9981 mdc.lustre-MDT0000-mdc-ffff913d02dae000.batch_stats= snapshot_time: 1787860225.737100087 secs.nsecs start_time: 1787860117.151082648 secs.nsecs elapsed_time: 108.586017439 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff913d02dae000.batch_stats= snapshot_time: 1787860225.737164816 secs.nsecs start_time: 1787860117.151397431 secs.nsecs elapsed_time: 108.585767385 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff913d02dae000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff913d02dae000.batch_stats=0 llite.lustre-ffff913d02dae000.statahead_max=2048 llite.lustre-ffff913d02dae000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff913d02dae000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff913d02dae000.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.65 seconds: 27.37 ops/second llite.lustre-ffff913d02dae000.statahead_max=0 101 real 0m2.594s user 0m0.003s sys 0m0.356s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff913d02dae000.statahead_max=2048 llite.lustre-ffff913d02dae000.statahead_max=2048 101 real 0m0.645s user 0m0.005s sys 0m0.163s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10080 miss_total: 20 swrong=0 ewrong=0 - open/close 711 (time 1787860538.56 total 10.01 last 71.02) total: 900 open/close in 12.58 seconds: 71.56 ops/second llite.lustre-ffff913d02dae000.statahead_max=0 1001 real 0m23.141s user 0m0.042s sys 0m3.104s 'ls -l' 1000 files without statahead: 24 sec llite.lustre-ffff913d02dae000.statahead_max=2048 llite.lustre-ffff913d02dae000.statahead_max=2048 1001 real 0m6.176s user 0m0.028s sys 0m1.606s 'ls -l' 1000 files with statahead: 6 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11079 miss_total: 21 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 21 seconds llite.lustre-ffff913d02dae000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11079 miss_total: 21 mdc.lustre-MDT0000-mdc-ffff913d02dae000.batch_stats= snapshot_time: 1787860604.674352581 secs.nsecs start_time: 1787860228.331131598 secs.nsecs elapsed_time: 376.343220983 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 20 20 8: 4 40 60 16: 1 10 70 32: 0 0 70 64: 1 10 80 128: 0 0 80 256: 2 20 100 mdc.lustre-MDT0001-mdc-ffff913d02dae000.batch_stats= snapshot_time: 1787860604.674434795 secs.nsecs start_time: 1787860228.331302758 secs.nsecs elapsed_time: 376.343132037 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 3 30 30 8: 1 10 40 16: 2 20 60 32: 2 20 80 64: 0 0 80 128: 0 0 80 256: 2 20 100 - open/close 753 (time 1787860617.51 total 10.01 last 75.20) - open/close 1456 (time 1787860627.51 total 20.02 last 70.26) - open/close 2204 (time 1787860637.51 total 30.02 last 74.78) - open/close 2897 (time 1787860647.52 total 40.02 last 69.29) - open/close 3587 (time 1787860657.52 total 50.02 last 69.00) - open/close 4363 (time 1787860667.52 total 60.03 last 77.53) - open/close 5155 (time 1787860677.54 total 70.04 last 79.11) - open/close 5783 (time 1787860687.55 total 80.06 last 62.69) - open/close 6445 (time 1787860697.56 total 90.07 last 66.13) - open/close 7167 (time 1787860707.57 total 100.07 last 72.17) - open/close 7853 (time 1787860717.57 total 110.07 last 68.60) - open/close 8536 (time 1787860727.57 total 120.08 last 68.28) - open/close 9252 (time 1787860737.57 total 130.08 last 71.59) total: 10000 open/close in 140.08 seconds: 71.39 ops/second llite.lustre-ffff913d02dae000.statahead_batch_max=0 llite.lustre-ffff913d02dae000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff913d02dae000.stats=clear mdc.lustre-MDT0001-mdc-ffff913d02dae000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m3.293s user 0m0.255s sys 0m14.462s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff913d02dae000.statahead_batch_max=256 llite.lustre-ffff913d02dae000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff913d02dae000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff913d02dae000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff913d02dae000.stats=clear mdc.lustre-MDT0001-mdc-ffff913d02dae000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m54.552s user 0m0.269s sys 0m14.043s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff913d02dae000.batch_stats= snapshot_time: 1787860882.436524109 secs.nsecs start_time: 1787860818.639760969 secs.nsecs elapsed_time: 63.796763140 secs.nsecs subreqs per batch batches %% cum %% 1: 73 35 35 2: 29 13 49 4: 23 11 60 8: 16 7 67 16: 7 3 71 32: 18 8 79 64: 6 2 82 128: 1 0 83 256: 35 16 100 mdc.lustre-MDT0001-mdc-ffff913d02dae000.batch_stats= snapshot_time: 1787860882.436582125 secs.nsecs start_time: 1787860818.639853544 secs.nsecs elapsed_time: 63.796728581 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff913d02dae000.statahead_batch_max=64 llite.lustre-ffff913d02dae000.statahead_max=128