== sanity test 123ad: Verify batching statahead works correctly ========================================================== 21:49:59 (1790732999) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffffa02491c07000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffffa02491c07000.batch_stats=0 llite.lustre-ffffa02491c07000.statahead_max=32 llite.lustre-ffffa02491c07000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffffa02491c07000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffffa02491c07000.batch_stats=0 striped dir -i1 -c2 -H all_char /mnt/lustre/d123ad.sanity total: 100 open/close in 2.99 seconds: 33.40 ops/second llite.lustre-ffffa02491c07000.statahead_max=0 101 real 0m2.872s user 0m0.003s sys 0m0.339s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffffa02491c07000.statahead_max=32 llite.lustre-ffffa02491c07000.statahead_max=32 101 real 0m1.036s user 0m0.002s sys 0m0.173s 'ls -l' 100 files with statahead: 1 sec statahead total: 117 statahead wrong: 2 agl total: 112 list_total: 31 fname_total: 72 hit_total: 4607 miss_total: 1282 swrong=2 ewrong=2 - open/close 647 (time 1790733027.77 total 10.01 last 64.66) total: 900 open/close in 13.94 seconds: 64.57 ops/second llite.lustre-ffffa02491c07000.statahead_max=0 1001 real 0m25.225s user 0m0.035s sys 0m3.083s 'ls -l' 1000 files without statahead: 25 sec llite.lustre-ffffa02491c07000.statahead_max=32 llite.lustre-ffffa02491c07000.statahead_max=32 1001 real 0m8.993s user 0m0.028s sys 0m1.346s 'ls -l' 1000 files with statahead: 9 sec statahead total: 118 statahead wrong: 2 agl total: 113 list_total: 32 fname_total: 72 hit_total: 5603 miss_total: 1286 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 24 seconds llite.lustre-ffffa02491c07000.statahead_stats= statahead total: 118 statahead wrong: 2 agl total: 113 list_total: 32 fname_total: 72 hit_total: 5603 miss_total: 1286 mdc.lustre-MDT0000-mdc-ffffa02491c07000.batch_stats= snapshot_time: 1790733102.713732596 secs.nsecs start_time: 1790733004.215557875 secs.nsecs elapsed_time: 98.498174721 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 3 7 10 8: 3 7 17 16: 20 50 67 32: 13 32 100 mdc.lustre-MDT0001-mdc-ffffa02491c07000.batch_stats= snapshot_time: 1790733102.713805750 secs.nsecs start_time: 1790733004.215714520 secs.nsecs elapsed_time: 98.498091230 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 4 10 10 8: 2 5 15 16: 23 58 74 32: 10 25 100 - open/close 674 (time 1790733115.26 total 10.00 last 67.38) - open/close 1364 (time 1790733125.26 total 20.01 last 68.97) - open/close 2028 (time 1790733135.26 total 30.01 last 66.38) - open/close 2736 (time 1790733145.27 total 40.01 last 70.79) - open/close 3409 (time 1790733155.29 total 50.03 last 67.16) - open/close 4026 (time 1790733165.29 total 60.04 last 61.67) - open/close 4664 (time 1790733175.31 total 70.06 last 63.67) - open/close 5284 (time 1790733185.32 total 80.07 last 61.94) - open/close 5914 (time 1790733195.33 total 90.08 last 62.94) - open/close 6531 (time 1790733205.35 total 100.09 last 61.61) - open/close 7121 (time 1790733215.36 total 110.11 last 58.90) - open/close 7789 (time 1790733225.37 total 120.11 last 66.77) - open/close 8523 (time 1790733235.37 total 130.11 last 73.39) - open/close 9274 (time 1790733245.37 total 140.11 last 75.10) - open/close 9974 (time 1790733255.38 total 150.12 last 69.94) total: 10000 open/close in 150.52 seconds: 66.43 ops/second llite.lustre-ffffa02491c07000.statahead_batch_max=0 llite.lustre-ffffa02491c07000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffffa02491c07000.stats=clear mdc.lustre-MDT0001-mdc-ffffa02491c07000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m56.546s user 0m0.312s sys 0m16.669s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffffa02491c07000.statahead_batch_max=32 llite.lustre-ffffa02491c07000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffffa02491c07000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffffa02491c07000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffffa02491c07000.stats=clear mdc.lustre-MDT0001-mdc-ffffa02491c07000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m40.288s user 0m0.299s sys 0m14.154s batched statahead RPC= hit count=9991 mdc.lustre-MDT0000-mdc-ffffa02491c07000.batch_stats= snapshot_time: 1790733430.134396145 secs.nsecs start_time: 1790733319.680194213 secs.nsecs elapsed_time: 110.454201932 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffffa02491c07000.batch_stats= snapshot_time: 1790733430.134461268 secs.nsecs start_time: 1790733319.680489723 secs.nsecs elapsed_time: 110.453971545 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffffa02491c07000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffffa02491c07000.batch_stats=0 llite.lustre-ffffa02491c07000.statahead_max=2048 llite.lustre-ffffa02491c07000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffffa02491c07000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffffa02491c07000.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.77 seconds: 36.14 ops/second llite.lustre-ffffa02491c07000.statahead_max=0 101 real 0m2.837s user 0m0.006s sys 0m0.373s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffffa02491c07000.statahead_max=2048 llite.lustre-ffffa02491c07000.statahead_max=2048 101 real 0m0.846s user 0m0.002s sys 0m0.160s 'ls -l' 100 files with statahead: 0 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10090 miss_total: 10 swrong=0 ewrong=0 - open/close 671 (time 1790733764.65 total 10.01 last 67.03) total: 900 open/close in 13.67 seconds: 65.84 ops/second llite.lustre-ffffa02491c07000.statahead_max=0 1001 real 0m29.163s user 0m0.029s sys 0m3.104s 'ls -l' 1000 files without statahead: 29 sec llite.lustre-ffffa02491c07000.statahead_max=2048 llite.lustre-ffffa02491c07000.statahead_max=2048 1001 real 0m6.883s user 0m0.030s sys 0m1.434s 'ls -l' 1000 files with statahead: 6 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11089 miss_total: 11 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 23 seconds llite.lustre-ffffa02491c07000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11089 miss_total: 11 mdc.lustre-MDT0000-mdc-ffffa02491c07000.batch_stats= snapshot_time: 1790733839.738147687 secs.nsecs start_time: 1790733432.266275345 secs.nsecs elapsed_time: 407.471872342 secs.nsecs subreqs per batch batches %% cum %% 1: 1 16 16 2: 0 0 16 4: 3 50 66 8: 0 0 66 16: 0 0 66 32: 0 0 66 64: 0 0 66 128: 0 0 66 256: 2 33 100 mdc.lustre-MDT0001-mdc-ffffa02491c07000.batch_stats= snapshot_time: 1790733839.738415194 secs.nsecs start_time: 1790733432.266451868 secs.nsecs elapsed_time: 407.471963326 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 11 11 8: 2 22 33 16: 1 11 44 32: 1 11 55 64: 2 22 77 128: 0 0 77 256: 2 22 100 - open/close 687 (time 1790733852.42 total 10.02 last 68.53) - open/close 1370 (time 1790733862.42 total 20.03 last 68.26) - open/close 2091 (time 1790733872.43 total 30.04 last 72.04) - open/close 2791 (time 1790733882.44 total 40.05 last 69.94) - open/close 3442 (time 1790733892.45 total 50.06 last 65.04) - open/close 4127 (time 1790733902.45 total 60.06 last 68.48) - open/close 4797 (time 1790733912.47 total 70.07 last 66.91) - open/close 5498 (time 1790733922.47 total 80.08 last 70.06) - open/close 6197 (time 1790733932.48 total 90.09 last 69.81) - open/close 6867 (time 1790733942.50 total 100.11 last 66.87) - open/close 7484 (time 1790733952.51 total 110.11 last 61.68) - open/close 8171 (time 1790733962.52 total 120.13 last 68.59) - open/close 8817 (time 1790733972.52 total 130.13 last 64.59) - open/close 9472 (time 1790733982.53 total 140.14 last 65.44) total: 10000 open/close in 147.98 seconds: 67.58 ops/second llite.lustre-ffffa02491c07000.statahead_batch_max=0 llite.lustre-ffffa02491c07000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffffa02491c07000.stats=clear mdc.lustre-MDT0001-mdc-ffffa02491c07000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m4.533s user 0m0.337s sys 0m15.770s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffffa02491c07000.statahead_batch_max=256 llite.lustre-ffffa02491c07000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffffa02491c07000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffffa02491c07000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffffa02491c07000.stats=clear mdc.lustre-MDT0001-mdc-ffffa02491c07000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m55.755s user 0m0.319s sys 0m15.484s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffffa02491c07000.batch_stats= snapshot_time: 1790734130.739788910 secs.nsecs start_time: 1790734063.924176933 secs.nsecs elapsed_time: 66.815611977 secs.nsecs subreqs per batch batches %% cum %% 1: 70 37 37 2: 23 12 50 4: 18 9 59 8: 9 4 64 16: 11 5 70 32: 11 5 76 64: 8 4 80 128: 1 0 81 256: 35 18 100 mdc.lustre-MDT0001-mdc-ffffa02491c07000.batch_stats= snapshot_time: 1790734130.739918685 secs.nsecs start_time: 1790734063.924535029 secs.nsecs elapsed_time: 66.815383656 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffffa02491c07000.statahead_batch_max=64 llite.lustre-ffffa02491c07000.statahead_max=128