== sanity test 123ad: Verify batching statahead works correctly ========================================================== 08:20:49 (1787833249) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff8fec713d8800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8fec713d8800.batch_stats=0 llite.lustre-ffff8fec713d8800.statahead_max=32 llite.lustre-ffff8fec713d8800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff8fec713d8800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8fec713d8800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.29 seconds: 30.42 ops/second llite.lustre-ffff8fec713d8800.statahead_max=0 101 real 0m2.815s user 0m0.004s sys 0m0.324s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff8fec713d8800.statahead_max=32 llite.lustre-ffff8fec713d8800.statahead_max=32 101 real 0m1.119s user 0m0.003s sys 0m0.159s 'ls -l' 100 files with statahead: 1 sec statahead total: 149 statahead wrong: 2 agl total: 143 list_total: 31 fname_total: 116 hit_total: 4626 miss_total: 1270 swrong=2 ewrong=2 - open/close 722 (time 1787833280.07 total 10.01 last 72.11) total: 900 open/close in 12.93 seconds: 69.59 ops/second llite.lustre-ffff8fec713d8800.statahead_max=0 1001 real 0m24.273s user 0m0.036s sys 0m3.049s 'ls -l' 1000 files without statahead: 24 sec llite.lustre-ffff8fec713d8800.statahead_max=32 llite.lustre-ffff8fec713d8800.statahead_max=32 1001 real 0m9.939s user 0m0.033s sys 0m1.485s 'ls -l' 1000 files with statahead: 9 sec statahead total: 150 statahead wrong: 2 agl total: 144 list_total: 32 fname_total: 116 hit_total: 5624 miss_total: 1272 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 25 seconds llite.lustre-ffff8fec713d8800.statahead_stats= statahead total: 150 statahead wrong: 2 agl total: 144 list_total: 32 fname_total: 116 hit_total: 5624 miss_total: 1272 mdc.lustre-MDT0000-mdc-ffff8fec713d8800.batch_stats= snapshot_time: 1787833357.014970390 secs.nsecs start_time: 1787833255.278814389 secs.nsecs elapsed_time: 101.736156001 secs.nsecs subreqs per batch batches %% cum %% 1: 5 12 12 2: 0 0 12 4: 2 5 17 8: 2 5 23 16: 23 58 82 32: 7 17 100 mdc.lustre-MDT0001-mdc-ffff8fec713d8800.batch_stats= snapshot_time: 1787833357.015045511 secs.nsecs start_time: 1787833255.278894133 secs.nsecs elapsed_time: 101.736151378 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 3 7 10 8: 3 7 17 16: 13 32 50 32: 20 50 100 - open/close 595 (time 1787833370.14 total 10.02 last 59.40) - open/close 1198 (time 1787833380.14 total 20.02 last 60.29) - open/close 1783 (time 1787833390.14 total 30.02 last 58.49) - open/close 2444 (time 1787833400.15 total 40.02 last 66.08) - open/close 3065 (time 1787833410.15 total 50.03 last 62.08) - open/close 3578 (time 1787833420.15 total 60.03 last 51.29) - open/close 4202 (time 1787833430.16 total 70.03 last 62.37) - open/close 4815 (time 1787833440.17 total 80.04 last 61.25) - open/close 5414 (time 1787833450.19 total 90.07 last 59.77) - open/close 6034 (time 1787833460.19 total 100.07 last 61.96) - open/close 6674 (time 1787833470.20 total 110.07 last 63.98) - open/close 7375 (time 1787833480.20 total 120.07 last 70.09) - open/close 8060 (time 1787833490.22 total 130.10 last 68.36) - open/close 8723 (time 1787833500.23 total 140.11 last 66.23) - open/close 9415 (time 1787833510.23 total 150.11 last 69.19) total: 10000 open/close in 157.47 seconds: 63.50 ops/second llite.lustre-ffff8fec713d8800.statahead_batch_max=0 llite.lustre-ffff8fec713d8800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8fec713d8800.stats=clear mdc.lustre-MDT0001-mdc-ffff8fec713d8800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m12.036s user 0m0.374s sys 0m16.407s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8fec713d8800.statahead_batch_max=32 llite.lustre-ffff8fec713d8800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8fec713d8800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8fec713d8800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8fec713d8800.stats=clear mdc.lustre-MDT0001-mdc-ffff8fec713d8800.stats=clear ls -l files in d123ad.sanity: 10001 real 2m10.394s user 0m0.375s sys 0m15.894s batched statahead RPC= hit count=9982 mdc.lustre-MDT0000-mdc-ffff8fec713d8800.batch_stats= snapshot_time: 1787833742.342176839 secs.nsecs start_time: 1787833599.249261726 secs.nsecs elapsed_time: 143.092915113 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff8fec713d8800.batch_stats= snapshot_time: 1787833742.342234514 secs.nsecs start_time: 1787833599.249409938 secs.nsecs elapsed_time: 143.092824576 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff8fec713d8800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8fec713d8800.batch_stats=0 llite.lustre-ffff8fec713d8800.statahead_max=2048 llite.lustre-ffff8fec713d8800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff8fec713d8800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8fec713d8800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.28 seconds: 30.46 ops/second llite.lustre-ffff8fec713d8800.statahead_max=0 101 real 0m2.976s user 0m0.005s sys 0m0.352s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff8fec713d8800.statahead_max=2048 llite.lustre-ffff8fec713d8800.statahead_max=2048 101 real 0m1.211s user 0m0.007s sys 0m0.194s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10081 miss_total: 19 swrong=0 ewrong=0 - open/close 644 (time 1787834079.73 total 10.01 last 64.33) total: 900 open/close in 14.35 seconds: 62.74 ops/second llite.lustre-ffff8fec713d8800.statahead_max=0 1001 real 0m27.351s user 0m0.044s sys 0m3.358s 'ls -l' 1000 files without statahead: 27 sec llite.lustre-ffff8fec713d8800.statahead_max=2048 llite.lustre-ffff8fec713d8800.statahead_max=2048 1001 real 0m8.087s user 0m0.040s sys 0m1.685s 'ls -l' 1000 files with statahead: 8 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11080 miss_total: 20 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 30 seconds llite.lustre-ffff8fec713d8800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11080 miss_total: 20 mdc.lustre-MDT0000-mdc-ffff8fec713d8800.batch_stats= snapshot_time: 1787834164.954299125 secs.nsecs start_time: 1787833745.602779484 secs.nsecs elapsed_time: 419.351519641 secs.nsecs subreqs per batch batches %% cum %% 1: 2 28 28 2: 0 0 28 4: 2 28 57 8: 1 14 71 16: 0 0 71 32: 0 0 71 64: 0 0 71 128: 0 0 71 256: 2 28 100 mdc.lustre-MDT0001-mdc-ffff8fec713d8800.batch_stats= snapshot_time: 1787834164.954369275 secs.nsecs start_time: 1787833745.603016670 secs.nsecs elapsed_time: 419.351352605 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 25 25 8: 2 25 50 16: 0 0 50 32: 0 0 50 64: 1 12 62 128: 1 12 75 256: 2 25 100 - open/close 524 (time 1787834179.24 total 10.01 last 52.37) - open/close 1019 (time 1787834189.25 total 20.01 last 49.49) - open/close 1498 (time 1787834199.30 total 30.06 last 47.64) - open/close 1934 (time 1787834209.30 total 40.06 last 43.60) - open/close 2426 (time 1787834219.30 total 50.07 last 49.19) - open/close 2897 (time 1787834229.31 total 60.07 last 47.06) - open/close 3508 (time 1787834239.32 total 70.08 last 61.08) - open/close 4117 (time 1787834249.32 total 80.08 last 60.87) - open/close 4716 (time 1787834259.33 total 90.10 last 59.81) - open/close 5262 (time 1787834269.35 total 100.11 last 54.53) - open/close 5741 (time 1787834279.35 total 110.11 last 47.88) - open/close 6287 (time 1787834289.37 total 120.13 last 54.52) - open/close 6799 (time 1787834299.37 total 130.13 last 51.20) - open/close 7301 (time 1787834309.37 total 140.13 last 50.18) - open/close 7839 (time 1787834319.38 total 150.14 last 53.77) - open/close 8283 (time 1787834329.39 total 160.15 last 44.34) - open/close 8922 (time 1787834339.40 total 170.16 last 63.85) - open/close 9532 (time 1787834349.40 total 180.16 last 60.99) total: 10000 open/close in 188.46 seconds: 53.06 ops/second llite.lustre-ffff8fec713d8800.statahead_batch_max=0 llite.lustre-ffff8fec713d8800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8fec713d8800.stats=clear mdc.lustre-MDT0001-mdc-ffff8fec713d8800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m25.089s user 0m0.473s sys 0m17.626s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8fec713d8800.statahead_batch_max=256 llite.lustre-ffff8fec713d8800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8fec713d8800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8fec713d8800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8fec713d8800.stats=clear mdc.lustre-MDT0001-mdc-ffff8fec713d8800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m2.959s user 0m0.381s sys 0m17.269s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff8fec713d8800.batch_stats= snapshot_time: 1787834531.090107918 secs.nsecs start_time: 1787834453.432970400 secs.nsecs elapsed_time: 77.657137518 secs.nsecs subreqs per batch batches %% cum %% 1: 155 50 50 2: 36 11 62 4: 27 8 71 8: 17 5 77 16: 14 4 81 32: 12 3 85 64: 9 2 88 128: 1 0 88 256: 34 11 100 mdc.lustre-MDT0001-mdc-ffff8fec713d8800.batch_stats= snapshot_time: 1787834531.090185558 secs.nsecs start_time: 1787834453.433190755 secs.nsecs elapsed_time: 77.656994803 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff8fec713d8800.statahead_batch_max=64 llite.lustre-ffff8fec713d8800.statahead_max=128