== sanity test 123ad: Verify batching statahead works correctly ========================================================== 19:19:05 (1712013545) batching: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff880083051800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff880083051800.batch_stats=0 llite.lustre-ffff880083051800.statahead_max=32 llite.lustre-ffff880083051800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff880083051800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff880083051800.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 1.50 seconds: 66.46 ops/second llite.lustre-ffff880083051800.statahead_max=0 101 real 0m1.302s user 0m0.008s sys 0m0.295s ls -l 100 files without statahead: 1 sec llite.lustre-ffff880083051800.statahead_max=32 32 101 real 0m0.481s user 0m0.005s sys 0m0.137s ls -l 100 files with statahead: 1 sec statahead total: 29 statahead wrong: 0 agl total: 29 list_total: 29 fname_total: 0 hit_total: 1600 miss_total: 140 - open/close 235 (time 1712013566.85 total 12.78 last 18.38) total: 900 open/close in 18.06 seconds: 49.84 ops/second llite.lustre-ffff880083051800.statahead_max=0 1001 real 0m12.668s user 0m0.094s sys 0m3.001s ls -l 1000 files without statahead: 12 sec llite.lustre-ffff880083051800.statahead_max=32 32 1001 real 0m2.865s user 0m0.037s sys 0m1.014s ls -l 1000 files with statahead: 3 sec statahead total: 30 statahead wrong: 0 agl total: 30 list_total: 30 fname_total: 0 hit_total: 2599 miss_total: 149 ls -l done rm -r /mnt/lustre/d123ad.sanity/: 9 seconds rm done statahead total: 30 statahead wrong: 0 agl total: 30 list_total: 30 fname_total: 0 hit_total: 2599 miss_total: 149 mdc.lustre-MDT0000-mdc-ffff880083051800.batch_stats= snapshot_time: 1712013601.273908926 secs.nsecs start_time: 1712013547.546312415 secs.nsecs elapsed_time: 53.727596511 secs.nsecs subreqs per batch batches % cum % 1: 2 4 4 2: 1 2 7 4: 3 7 14 8: 2 4 19 16: 20 48 68 32: 13 31 100 mdc.lustre-MDT0001-mdc-ffff880083051800.batch_stats= snapshot_time: 1712013601.274012990 secs.nsecs start_time: 1712013547.547231091 secs.nsecs elapsed_time: 53.726781899 secs.nsecs subreqs per batch batches % cum % 1: 1 2 2 2: 0 0 2 4: 3 7 10 8: 1 2 12 16: 20 51 64 32: 14 35 100 - open/close 247 (time 1712013616.17 total 13.22 last 18.69) - open/close 1247 (time 1712013636.02 total 33.07 last 50.38) - open/close 1820 (time 1712013652.51 total 49.55 last 34.75) - open/close 2140 (time 1712013666.82 total 63.86 last 22.36) - open/close 3187 (time 1712013676.83 total 73.87 last 104.60) - open/close 3371 (time 1712013689.63 total 86.68 last 14.37) - open/close 3454 (time 1712013701.47 total 98.51 last 7.02) - open/close 3993 (time 1712013718.67 total 115.71 last 31.33) - open/close 4262 (time 1712013732.73 total 129.78 last 19.13) - open/close 5383 (time 1712013742.73 total 139.78 last 112.08) - open/close 6410 (time 1712013752.74 total 149.78 last 102.67) - open/close 7452 (time 1712013762.74 total 159.78 last 104.18) - open/close 8315 (time 1712013772.75 total 169.80 last 86.18) - open/close 9211 (time 1712013782.75 total 179.80 last 89.60) total: 10000 open/close in 189.65 seconds: 52.73 ops/second llite.lustre-ffff880083051800.statahead_batch_max=0 llite.lustre-ffff880083051800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff880083051800.stats=clear mdc.lustre-MDT0001-mdc-ffff880083051800.stats=clear 10001 real 0m21.925s user 0m0.330s sys 0m12.360s llite.lustre-ffff880083051800.statahead_batch_max=32 llite.lustre-ffff880083051800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff880083051800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff880083051800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff880083051800.stats=clear mdc.lustre-MDT0001-mdc-ffff880083051800.stats=clear 10001 real 0m46.764s user 0m0.388s sys 0m14.072s unbatched RPCs: 10004, batched RPCs: 316 mdc.lustre-MDT0000-mdc-ffff880083051800.batch_stats= snapshot_time: 1712013874.712001576 secs.nsecs start_time: 1712013819.956400736 secs.nsecs elapsed_time: 54.755600840 secs.nsecs subreqs per batch batches % cum % 1: 1 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 2 0 1 32: 312 98 100 mdc.lustre-MDT0001-mdc-ffff880083051800.batch_stats= snapshot_time: 1712013874.712469115 secs.nsecs start_time: 1712013819.956535317 secs.nsecs elapsed_time: 54.755933798 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 batching: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff880083051800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff880083051800.batch_stats=0 llite.lustre-ffff880083051800.statahead_max=2048 llite.lustre-ffff880083051800.statahead_batch_max=256 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff880083051800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff880083051800.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 1.87 seconds: 53.47 ops/second llite.lustre-ffff880083051800.statahead_max=0 101 real 0m1.322s user 0m0.007s sys 0m0.324s ls -l 100 files without statahead: 1 sec llite.lustre-ffff880083051800.statahead_max=2048 2048 101 real 0m0.310s user 0m0.006s sys 0m0.125s ls -l 100 files with statahead: 0 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10098 miss_total: 55 total: 900 open/close in 6.69 seconds: 134.49 ops/second llite.lustre-ffff880083051800.statahead_max=0 1001 real 0m12.693s user 0m0.119s sys 0m2.971s ls -l 1000 files without statahead: 12 sec llite.lustre-ffff880083051800.statahead_max=2048 2048 1001 real 0m2.318s user 0m0.036s sys 0m1.086s ls -l 1000 files with statahead: 2 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11097 miss_total: 56 ls -l done rm -r /mnt/lustre/d123ad.sanity/: 9 seconds rm done statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11097 miss_total: 56 mdc.lustre-MDT0000-mdc-ffff880083051800.batch_stats= snapshot_time: 1712014034.633520239 secs.nsecs start_time: 1712013876.023978281 secs.nsecs elapsed_time: 158.609541958 secs.nsecs subreqs per batch batches % cum % 1: 1 14 14 2: 0 0 14 4: 2 28 42 8: 2 28 71 16: 0 0 71 32: 0 0 71 64: 0 0 71 128: 0 0 71 256: 2 28 100 mdc.lustre-MDT0001-mdc-ffff880083051800.batch_stats= snapshot_time: 1712014034.635327279 secs.nsecs start_time: 1712013876.024146155 secs.nsecs elapsed_time: 158.611181124 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 2: 0 0 0 4: 2 22 22 8: 2 22 44 16: 1 11 55 32: 1 11 66 64: 0 0 66 128: 1 11 77 256: 2 22 100 - open/close 1378 (time 1712014045.82 total 10.00 last 137.74) - open/close 2603 (time 1712014055.82 total 20.00 last 122.50) - open/close 3469 (time 1712014075.34 total 39.52 last 44.38) - open/close 3563 (time 1712014087.51 total 51.69 last 7.72) - open/close 3694 (time 1712014100.18 total 64.36 last 10.34) - open/close 4171 (time 1712014116.85 total 81.03 last 28.61) - open/close 4267 (time 1712014128.83 total 93.01 last 8.01) - open/close 4938 (time 1712014146.97 total 111.15 last 36.98) - open/close 5022 (time 1712014158.82 total 123.00 last 7.09) - open/close 5168 (time 1712014171.51 total 135.69 last 11.50) - open/close 5296 (time 1712014183.88 total 148.06 last 10.35) - open/close 6215 (time 1712014193.88 total 158.06 last 91.89) - open/close 7034 (time 1712014203.89 total 168.07 last 81.80) - open/close 7660 (time 1712014223.37 total 187.55 last 32.14) - open/close 8400 (time 1712014233.38 total 197.56 last 73.91) - open/close 9125 (time 1712014243.39 total 207.57 last 72.46) - open/close 9636 (time 1712014260.55 total 224.73 last 29.78) - open/close 9742 (time 1712014273.14 total 237.32 last 8.42) - open/close 9833 (time 1712014285.37 total 249.55 last 7.44) total: 10000 open/close in 252.14 seconds: 39.66 ops/second llite.lustre-ffff880083051800.statahead_batch_max=0 llite.lustre-ffff880083051800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff880083051800.stats=clear mdc.lustre-MDT0001-mdc-ffff880083051800.stats=clear 10001 real 0m32.295s user 0m0.336s sys 0m14.476s llite.lustre-ffff880083051800.statahead_batch_max=256 llite.lustre-ffff880083051800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff880083051800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff880083051800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff880083051800.stats=clear mdc.lustre-MDT0001-mdc-ffff880083051800.stats=clear 10001 real 0m27.298s user 0m0.404s sys 0m16.136s unbatched RPCs: 10004, batched RPCs: 195 mdc.lustre-MDT0000-mdc-ffff880083051800.batch_stats= snapshot_time: 1712014364.354884136 secs.nsecs start_time: 1712014328.171832863 secs.nsecs elapsed_time: 36.183051273 secs.nsecs subreqs per batch batches % cum % 1: 89 45 45 2: 6 3 48 4: 12 6 54 8: 17 8 63 16: 13 6 70 32: 14 7 77 64: 9 4 82 128: 0 0 82 256: 35 17 100 mdc.lustre-MDT0001-mdc-ffff880083051800.batch_stats= snapshot_time: 1712014364.355054596 secs.nsecs start_time: 1712014328.172671121 secs.nsecs elapsed_time: 36.182383475 secs.nsecs subreqs per batch batches % cum % 1: 0 0 0 seq.cli-lustre-OST0000-super.width=33554431 seq.cli-lustre-OST0001-super.width=33554431 seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff880083051800.statahead_batch_max=64 llite.lustre-ffff880083051800.statahead_max=128