== sanity test 123ad: Verify batching statahead works correctly ========================================================== 10:29:38 (1785940178) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9852d126f800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9852d126f800.batch_stats=0 llite.lustre-ffff9852d126f800.statahead_max=32 llite.lustre-ffff9852d126f800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9852d126f800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9852d126f800.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.96 seconds: 25.23 ops/second llite.lustre-ffff9852d126f800.statahead_max=0 101 real 0m2.675s user 0m0.002s sys 0m0.326s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff9852d126f800.statahead_max=32 llite.lustre-ffff9852d126f800.statahead_max=32 101 real 0m1.295s user 0m0.005s sys 0m0.149s 'ls -l' 100 files with statahead: 1 sec statahead total: 182 statahead wrong: 2 agl total: 174 list_total: 32 fname_total: 150 hit_total: 3650 miss_total: 1582 swrong=2 ewrong=2 - open/close 694 (time 1785940212.26 total 10.00 last 69.40) total: 900 open/close in 14.60 seconds: 61.62 ops/second llite.lustre-ffff9852d126f800.statahead_max=0 1001 real 0m25.730s user 0m0.035s sys 0m2.701s 'ls -l' 1000 files without statahead: 25 sec llite.lustre-ffff9852d126f800.statahead_max=32 llite.lustre-ffff9852d126f800.statahead_max=32 1001 real 0m10.646s user 0m0.027s sys 0m1.322s 'ls -l' 1000 files with statahead: 11 sec statahead total: 183 statahead wrong: 2 agl total: 175 list_total: 33 fname_total: 150 hit_total: 4648 miss_total: 1584 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 28 seconds llite.lustre-ffff9852d126f800.statahead_stats= statahead total: 183 statahead wrong: 2 agl total: 175 list_total: 33 fname_total: 150 hit_total: 4648 miss_total: 1584 mdc.lustre-MDT0000-mdc-ffff9852d126f800.batch_stats= snapshot_time: 1785940296.917878399 secs.nsecs start_time: 1785940185.857107002 secs.nsecs elapsed_time: 111.060771397 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 1 2 2 4: 2 5 7 8: 4 10 17 16: 20 51 69 32: 12 30 100 mdc.lustre-MDT0001-mdc-ffff9852d126f800.batch_stats= snapshot_time: 1785940296.917933094 secs.nsecs start_time: 1785940185.857280630 secs.nsecs elapsed_time: 111.060652464 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 1 2 5 4: 2 5 10 8: 2 5 15 16: 20 50 65 32: 14 35 100 - open/close 558 (time 1785940309.92 total 10.02 last 55.69) - open/close 1204 (time 1785940319.93 total 20.03 last 64.51) - open/close 1764 (time 1785940329.95 total 30.05 last 55.88) - open/close 2328 (time 1785940339.97 total 40.07 last 56.32) - open/close 2802 (time 1785940349.97 total 50.07 last 47.37) - open/close 3224 (time 1785940359.98 total 60.08 last 42.18) - open/close 3600 (time 1785940370.00 total 70.10 last 37.53) - open/close 4275 (time 1785940380.03 total 80.13 last 67.29) - open/close 4787 (time 1785940390.04 total 90.14 last 51.14) - open/close 5440 (time 1785940400.04 total 100.14 last 65.28) - open/close 6211 (time 1785940410.05 total 110.15 last 77.05) - open/close 6925 (time 1785940420.05 total 120.15 last 71.38) - open/close 7446 (time 1785940430.07 total 130.17 last 52.03) - open/close 7939 (time 1785940440.29 total 140.39 last 48.23) - open/close 8393 (time 1785940450.31 total 150.41 last 45.29) - open/close 8829 (time 1785940460.34 total 160.44 last 43.49) - open/close 9333 (time 1785940470.37 total 170.47 last 50.25) - open/close 9773 (time 1785940480.37 total 180.47 last 43.98) total: 10000 open/close in 185.70 seconds: 53.85 ops/second llite.lustre-ffff9852d126f800.statahead_batch_max=0 llite.lustre-ffff9852d126f800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9852d126f800.stats=clear mdc.lustre-MDT0001-mdc-ffff9852d126f800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m29.496s user 0m0.270s sys 0m13.322s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9852d126f800.statahead_batch_max=32 llite.lustre-ffff9852d126f800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9852d126f800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9852d126f800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9852d126f800.stats=clear mdc.lustre-MDT0001-mdc-ffff9852d126f800.stats=clear ls -l files in d123ad.sanity: 10001 real 2m9.362s user 0m0.257s sys 0m12.567s batched statahead RPC= hit count=9981 mdc.lustre-MDT0000-mdc-ffff9852d126f800.batch_stats= snapshot_time: 1785940736.616243749 secs.nsecs start_time: 1785940591.095166001 secs.nsecs elapsed_time: 145.521077748 secs.nsecs subreqs per batch batches %% cum %% 1: 1 0 0 2: 1 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 1 32: 313 98 100 mdc.lustre-MDT0001-mdc-ffff9852d126f800.batch_stats= snapshot_time: 1785940736.616302420 secs.nsecs start_time: 1785940591.095294525 secs.nsecs elapsed_time: 145.521007895 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9852d126f800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9852d126f800.batch_stats=0 llite.lustre-ffff9852d126f800.statahead_max=2048 llite.lustre-ffff9852d126f800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9852d126f800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9852d126f800.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 4.09 seconds: 24.44 ops/second llite.lustre-ffff9852d126f800.statahead_max=0 101 real 0m2.852s user 0m0.006s sys 0m0.312s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9852d126f800.statahead_max=2048 llite.lustre-ffff9852d126f800.statahead_max=2048 101 real 0m1.166s user 0m0.006s sys 0m0.150s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10080 miss_total: 20 swrong=0 ewrong=0 - open/close 738 (time 1785941116.58 total 10.02 last 73.67) total: 900 open/close in 13.35 seconds: 67.44 ops/second llite.lustre-ffff9852d126f800.statahead_max=0 1001 real 0m32.601s user 0m0.029s sys 0m2.849s 'ls -l' 1000 files without statahead: 33 sec llite.lustre-ffff9852d126f800.statahead_max=2048 llite.lustre-ffff9852d126f800.statahead_max=2048 1001 real 0m9.854s user 0m0.021s sys 0m1.231s 'ls -l' 1000 files with statahead: 10 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11079 miss_total: 21 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 33 seconds llite.lustre-ffff9852d126f800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11079 miss_total: 21 mdc.lustre-MDT0000-mdc-ffff9852d126f800.batch_stats= snapshot_time: 1785941216.172876556 secs.nsecs start_time: 1785940739.240449738 secs.nsecs elapsed_time: 476.932426818 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 2 28 28 4: 1 14 42 8: 1 14 57 16: 1 14 71 32: 0 0 71 64: 0 0 71 128: 0 0 71 256: 2 28 100 mdc.lustre-MDT0001-mdc-ffff9852d126f800.batch_stats= snapshot_time: 1785941216.172945276 secs.nsecs start_time: 1785940739.240587158 secs.nsecs elapsed_time: 476.932358118 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 10 10 8: 3 30 40 16: 1 10 50 32: 2 20 70 64: 1 10 80 128: 0 0 80 256: 2 20 100 - open/close 455 (time 1785941229.41 total 10.00 last 45.48) - open/close 1181 (time 1785941239.41 total 20.01 last 72.58) - open/close 1699 (time 1785941249.42 total 30.01 last 51.77) - open/close 2336 (time 1785941259.42 total 40.02 last 63.67) - open/close 2759 (time 1785941269.43 total 50.02 last 42.28) - open/close 3290 (time 1785941279.45 total 60.05 last 52.96) - open/close 3945 (time 1785941289.46 total 70.06 last 65.46) - open/close 4425 (time 1785941299.47 total 80.07 last 47.95) - open/close 5062 (time 1785941309.48 total 90.07 last 63.66) - open/close 5753 (time 1785941319.48 total 100.08 last 69.07) - open/close 6515 (time 1785941329.49 total 110.09 last 76.10) - open/close 7155 (time 1785941339.50 total 120.09 last 63.99) - open/close 7867 (time 1785941349.50 total 130.10 last 71.16) - open/close 8386 (time 1785941359.51 total 140.10 last 51.87) - open/close 8865 (time 1785941369.53 total 150.13 last 47.78) - open/close 9208 (time 1785941379.54 total 160.13 last 34.28) - open/close 9629 (time 1785941389.57 total 170.16 last 41.98) total: 10000 open/close in 176.90 seconds: 56.53 ops/second llite.lustre-ffff9852d126f800.statahead_batch_max=0 llite.lustre-ffff9852d126f800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9852d126f800.stats=clear mdc.lustre-MDT0001-mdc-ffff9852d126f800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m22.342s user 0m0.264s sys 0m13.105s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9852d126f800.statahead_batch_max=256 llite.lustre-ffff9852d126f800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9852d126f800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9852d126f800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9852d126f800.stats=clear mdc.lustre-MDT0001-mdc-ffff9852d126f800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m9.265s user 0m0.267s sys 0m12.708s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9852d126f800.batch_stats= snapshot_time: 1785941576.731368185 secs.nsecs start_time: 1785941490.904981721 secs.nsecs elapsed_time: 85.826386464 secs.nsecs subreqs per batch batches %% cum %% 1: 57 32 32 2: 32 18 51 4: 8 4 55 8: 8 4 60 16: 6 3 63 32: 24 13 77 64: 3 1 79 128: 1 0 79 256: 35 20 100 mdc.lustre-MDT0001-mdc-ffff9852d126f800.batch_stats= snapshot_time: 1785941576.731516385 secs.nsecs start_time: 1785941490.905050037 secs.nsecs elapsed_time: 85.826466348 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9852d126f800.statahead_batch_max=64 llite.lustre-ffff9852d126f800.statahead_max=128