== sanity test 123ad: Verify batching statahead works correctly ========================================================== 14:36:59 (1785782219) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff8bd90874d800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8bd90874d800.batch_stats=0 llite.lustre-ffff8bd90874d800.statahead_max=32 llite.lustre-ffff8bd90874d800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff8bd90874d800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8bd90874d800.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.22 seconds: 31.10 ops/second llite.lustre-ffff8bd90874d800.statahead_max=0 101 real 0m4.172s user 0m0.005s sys 0m0.345s 'ls -l' 100 files without statahead: 4 sec llite.lustre-ffff8bd90874d800.statahead_max=32 llite.lustre-ffff8bd90874d800.statahead_max=32 101 real 0m1.882s user 0m0.004s sys 0m0.204s 'ls -l' 100 files with statahead: 2 sec statahead total: 153 statahead wrong: 2 agl total: 145 list_total: 32 fname_total: 118 hit_total: 4493 miss_total: 1292 swrong=2 ewrong=2 - open/close 464 (time 1785782258.46 total 10.01 last 46.36) total: 900 open/close in 16.32 seconds: 55.15 ops/second llite.lustre-ffff8bd90874d800.statahead_max=0 1001 real 0m33.687s user 0m0.046s sys 0m3.286s 'ls -l' 1000 files without statahead: 34 sec llite.lustre-ffff8bd90874d800.statahead_max=32 llite.lustre-ffff8bd90874d800.statahead_max=32 1001 real 0m16.720s user 0m0.031s sys 0m1.689s 'ls -l' 1000 files with statahead: 16 sec statahead total: 154 statahead wrong: 2 agl total: 146 list_total: 33 fname_total: 118 hit_total: 5486 miss_total: 1299 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 29 seconds llite.lustre-ffff8bd90874d800.statahead_stats= statahead total: 154 statahead wrong: 2 agl total: 146 list_total: 33 fname_total: 118 hit_total: 5486 miss_total: 1299 mdc.lustre-MDT0000-mdc-ffff8bd90874d800.batch_stats= snapshot_time: 1785782363.027118241 secs.nsecs start_time: 1785782225.399267038 secs.nsecs elapsed_time: 137.627851203 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 2 5 7 8: 3 7 15 16: 21 52 67 32: 13 32 100 mdc.lustre-MDT0001-mdc-ffff8bd90874d800.batch_stats= snapshot_time: 1785782363.027197535 secs.nsecs start_time: 1785782225.399389358 secs.nsecs elapsed_time: 137.627808177 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 1 2 5 4: 3 7 12 8: 3 7 20 16: 17 42 62 32: 15 37 100 - open/close 616 (time 1785782376.42 total 10.01 last 61.51) - open/close 1267 (time 1785782386.44 total 20.03 last 64.97) - open/close 1752 (time 1785782396.46 total 30.05 last 48.41) - open/close 2436 (time 1785782406.47 total 40.06 last 68.33) - open/close 3008 (time 1785782416.48 total 50.08 last 57.12) - open/close 3470 (time 1785782426.51 total 60.11 last 46.07) - open/close 4094 (time 1785782436.52 total 70.11 last 62.36) - open/close 4666 (time 1785782446.53 total 80.13 last 57.10) - open/close 5220 (time 1785782456.54 total 90.13 last 55.38) - open/close 5581 (time 1785782466.55 total 100.15 last 36.06) - open/close 6152 (time 1785782476.56 total 110.16 last 57.02) - open/close 6673 (time 1785782486.57 total 120.17 last 52.06) - open/close 7078 (time 1785782496.58 total 130.18 last 40.46) - open/close 7568 (time 1785782506.58 total 140.18 last 48.99) - open/close 8035 (time 1785782516.59 total 150.18 last 46.68) - open/close 8475 (time 1785782526.61 total 160.20 last 43.91) - open/close 8869 (time 1785782536.61 total 170.21 last 39.38) - open/close 9539 (time 1785782546.61 total 180.21 last 67.00) total: 10000 open/close in 189.01 seconds: 52.91 ops/second llite.lustre-ffff8bd90874d800.statahead_batch_max=0 llite.lustre-ffff8bd90874d800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8bd90874d800.stats=clear mdc.lustre-MDT0001-mdc-ffff8bd90874d800.stats=clear ls -l files in d123ad.sanity: 10001 real 2m0.315s user 0m0.338s sys 0m17.112s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8bd90874d800.statahead_batch_max=32 llite.lustre-ffff8bd90874d800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8bd90874d800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8bd90874d800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8bd90874d800.stats=clear mdc.lustre-MDT0001-mdc-ffff8bd90874d800.stats=clear ls -l files in d123ad.sanity: 10001 real 2m58.595s user 0m0.333s sys 0m16.168s batched statahead RPC= hit count=9975 mdc.lustre-MDT0000-mdc-ffff8bd90874d800.batch_stats= snapshot_time: 1785782897.810348111 secs.nsecs start_time: 1785782692.578122575 secs.nsecs elapsed_time: 205.232225536 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 2 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff8bd90874d800.batch_stats= snapshot_time: 1785782897.810425786 secs.nsecs start_time: 1785782692.578318517 secs.nsecs elapsed_time: 205.232107269 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff8bd90874d800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8bd90874d800.batch_stats=0 llite.lustre-ffff8bd90874d800.statahead_max=2048 llite.lustre-ffff8bd90874d800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff8bd90874d800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8bd90874d800.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 4.57 seconds: 21.87 ops/second llite.lustre-ffff8bd90874d800.statahead_max=0 101 real 0m4.210s user 0m0.006s sys 0m0.404s 'ls -l' 100 files without statahead: 4 sec llite.lustre-ffff8bd90874d800.statahead_max=2048 llite.lustre-ffff8bd90874d800.statahead_max=2048 101 real 0m1.768s user 0m0.001s sys 0m0.222s 'ls -l' 100 files with statahead: 2 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10074 miss_total: 26 swrong=0 ewrong=0 - open/close 475 (time 1785783381.75 total 10.03 last 47.36) total: 900 open/close in 19.48 seconds: 46.20 ops/second llite.lustre-ffff8bd90874d800.statahead_max=0 1001 real 0m38.440s user 0m0.027s sys 0m3.240s 'ls -l' 1000 files without statahead: 39 sec llite.lustre-ffff8bd90874d800.statahead_max=2048 llite.lustre-ffff8bd90874d800.statahead_max=2048 1001 real 0m9.358s user 0m0.022s sys 0m1.551s 'ls -l' 1000 files with statahead: 10 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11073 miss_total: 27 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 45 seconds llite.lustre-ffff8bd90874d800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11073 miss_total: 27 mdc.lustre-MDT0000-mdc-ffff8bd90874d800.batch_stats= snapshot_time: 1785783505.182528828 secs.nsecs start_time: 1785782901.775723477 secs.nsecs elapsed_time: 603.406805351 secs.nsecs subreqs per batch batches %% cum %% 1: 3 33 33 2: 1 11 44 4: 1 11 55 8: 1 11 66 16: 0 0 66 32: 1 11 77 64: 0 0 77 128: 0 0 77 256: 2 22 100 mdc.lustre-MDT0001-mdc-ffff8bd90874d800.batch_stats= snapshot_time: 1785783505.182594292 secs.nsecs start_time: 1785782901.775875223 secs.nsecs elapsed_time: 603.406719069 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 9 9 8: 3 27 36 16: 2 18 54 32: 3 27 81 64: 0 0 81 128: 0 0 81 256: 2 18 100 - open/close 457 (time 1785783520.22 total 10.01 last 45.64) - open/close 918 (time 1785783530.23 total 20.02 last 46.08) - open/close 1417 (time 1785783540.24 total 30.04 last 49.81) - open/close 1879 (time 1785783550.26 total 40.05 last 46.12) - open/close 2419 (time 1785783560.27 total 50.06 last 53.97) - open/close 2763 (time 1785783570.28 total 60.07 last 34.36) - open/close 3177 (time 1785783580.30 total 70.09 last 41.33) - open/close 3559 (time 1785783590.30 total 80.09 last 38.17) - open/close 3913 (time 1785783600.32 total 90.11 last 35.34) - open/close 4291 (time 1785783610.32 total 100.12 last 37.79) - open/close 4791 (time 1785783620.33 total 110.12 last 49.98) - open/close 5578 (time 1785783630.33 total 120.12 last 78.70) - open/close 6099 (time 1785783640.35 total 130.14 last 52.01) - open/close 6469 (time 1785783650.35 total 140.15 last 36.97) - open/close 6863 (time 1785783660.40 total 150.19 last 39.24) - open/close 7244 (time 1785783670.40 total 160.20 last 38.06) - open/close 7607 (time 1785783680.45 total 170.24 last 36.13) - open/close 8024 (time 1785783690.45 total 180.24 last 41.69) - open/close 8382 (time 1785783700.47 total 190.26 last 35.73) - open/close 8695 (time 1785783710.49 total 200.28 last 31.25) - open/close 9109 (time 1785783720.49 total 210.28 last 41.38) - open/close 9439 (time 1785783730.51 total 220.30 last 32.95) - open/close 9764 (time 1785783740.51 total 230.31 last 32.49) total: 10000 open/close in 233.78 seconds: 42.78 ops/second llite.lustre-ffff8bd90874d800.statahead_batch_max=0 llite.lustre-ffff8bd90874d800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8bd90874d800.stats=clear mdc.lustre-MDT0001-mdc-ffff8bd90874d800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m26.354s user 0m0.310s sys 0m15.789s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8bd90874d800.statahead_batch_max=256 llite.lustre-ffff8bd90874d800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8bd90874d800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8bd90874d800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8bd90874d800.stats=clear mdc.lustre-MDT0001-mdc-ffff8bd90874d800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m19.819s user 0m0.325s sys 0m15.319s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff8bd90874d800.batch_stats= snapshot_time: 1785783949.230196901 secs.nsecs start_time: 1785783847.506720305 secs.nsecs elapsed_time: 101.723476596 secs.nsecs subreqs per batch batches %% cum %% 1: 161 43 43 2: 78 21 64 4: 38 10 74 8: 18 4 79 16: 16 4 83 32: 20 5 89 64: 6 1 90 128: 1 0 91 256: 33 8 100 mdc.lustre-MDT0001-mdc-ffff8bd90874d800.batch_stats= snapshot_time: 1785783949.230263202 secs.nsecs start_time: 1785783847.506810238 secs.nsecs elapsed_time: 101.723452964 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff8bd90874d800.statahead_batch_max=64 llite.lustre-ffff8bd90874d800.statahead_max=128