== sanity test 123ad: Verify batching statahead works correctly ========================================================== 03:18:27 (1790925507) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.batch_stats=0 llite.lustre-ffff9a8e502b0800.statahead_max=32 llite.lustre-ffff9a8e502b0800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 3.44 seconds: 29.05 ops/second llite.lustre-ffff9a8e502b0800.statahead_max=0 101 real 0m2.962s user 0m0.007s sys 0m0.355s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff9a8e502b0800.statahead_max=32 llite.lustre-ffff9a8e502b0800.statahead_max=32 101 real 0m1.280s user 0m0.005s sys 0m0.177s 'ls -l' 100 files with statahead: 1 sec statahead total: 147 statahead wrong: 2 agl total: 140 list_total: 32 fname_total: 87 hit_total: 3948 miss_total: 1458 swrong=2 ewrong=2 - open/close 538 (time 1790925537.55 total 10.03 last 53.63) total: 900 open/close in 16.65 seconds: 54.06 ops/second llite.lustre-ffff9a8e502b0800.statahead_max=0 1001 real 0m27.364s user 0m0.037s sys 0m3.038s 'ls -l' 1000 files without statahead: 27 sec llite.lustre-ffff9a8e502b0800.statahead_max=32 llite.lustre-ffff9a8e502b0800.statahead_max=32 1001 real 0m10.066s user 0m0.027s sys 0m1.415s 'ls -l' 1000 files with statahead: 10 sec statahead total: 148 statahead wrong: 2 agl total: 141 list_total: 33 fname_total: 87 hit_total: 4946 miss_total: 1460 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 24 seconds llite.lustre-ffff9a8e502b0800.statahead_stats= statahead total: 148 statahead wrong: 2 agl total: 141 list_total: 33 fname_total: 87 hit_total: 4946 miss_total: 1460 mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.batch_stats= snapshot_time: 1790925620.137828059 secs.nsecs start_time: 1790925512.504872554 secs.nsecs elapsed_time: 107.632955505 secs.nsecs subreqs per batch batches %% cum %% 1: 4 10 10 2: 2 5 15 4: 1 2 17 8: 1 2 20 16: 26 66 87 32: 5 12 100 mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.batch_stats= snapshot_time: 1790925620.137904955 secs.nsecs start_time: 1790925512.505214957 secs.nsecs elapsed_time: 107.632689998 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 3 7 7 8: 4 10 17 16: 8 20 37 32: 25 62 100 - open/close 663 (time 1790925633.03 total 10.01 last 66.26) - open/close 1321 (time 1790925643.05 total 20.02 last 65.70) - open/close 1958 (time 1790925653.05 total 30.03 last 63.66) - open/close 2571 (time 1790925663.06 total 40.03 last 61.28) - open/close 3101 (time 1790925673.09 total 50.06 last 52.85) - open/close 3677 (time 1790925683.10 total 60.07 last 57.52) - open/close 4309 (time 1790925693.11 total 70.09 last 63.11) - open/close 4872 (time 1790925703.13 total 80.10 last 56.23) - open/close 5527 (time 1790925713.14 total 90.11 last 65.42) - open/close 6087 (time 1790925723.15 total 100.12 last 55.96) - open/close 6601 (time 1790925733.20 total 110.17 last 51.12) - open/close 7176 (time 1790925743.21 total 120.18 last 57.45) - open/close 7661 (time 1790925753.22 total 130.19 last 48.47) - open/close 8305 (time 1790925763.22 total 140.19 last 64.40) - open/close 9010 (time 1790925773.23 total 150.20 last 70.41) - open/close 9689 (time 1790925783.23 total 160.20 last 67.89) total: 10000 open/close in 164.81 seconds: 60.67 ops/second llite.lustre-ffff9a8e502b0800.statahead_batch_max=0 llite.lustre-ffff9a8e502b0800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.stats=clear mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m57.435s user 0m0.335s sys 0m16.485s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9a8e502b0800.statahead_batch_max=32 llite.lustre-ffff9a8e502b0800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.stats=clear mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m53.442s user 0m0.332s sys 0m14.711s batched statahead RPC= hit count=9982 mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.batch_stats= snapshot_time: 1790925978.078372062 secs.nsecs start_time: 1790925854.848270225 secs.nsecs elapsed_time: 123.230101837 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.batch_stats= snapshot_time: 1790925978.078442643 secs.nsecs start_time: 1790925854.848399733 secs.nsecs elapsed_time: 123.230042910 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.batch_stats=0 llite.lustre-ffff9a8e502b0800.statahead_max=2048 llite.lustre-ffff9a8e502b0800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.69 seconds: 27.08 ops/second llite.lustre-ffff9a8e502b0800.statahead_max=0 101 real 0m3.174s user 0m0.006s sys 0m0.339s 'ls -l' 100 files without statahead: 4 sec llite.lustre-ffff9a8e502b0800.statahead_max=2048 llite.lustre-ffff9a8e502b0800.statahead_max=2048 101 real 0m1.025s user 0m0.005s sys 0m0.182s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10081 miss_total: 19 swrong=0 ewrong=0 - open/close 610 (time 1790926358.84 total 10.02 last 60.88) total: 900 open/close in 15.54 seconds: 57.91 ops/second llite.lustre-ffff9a8e502b0800.statahead_max=0 1001 real 0m30.739s user 0m0.039s sys 0m3.251s 'ls -l' 1000 files without statahead: 30 sec llite.lustre-ffff9a8e502b0800.statahead_max=2048 llite.lustre-ffff9a8e502b0800.statahead_max=2048 1001 real 0m7.664s user 0m0.035s sys 0m1.410s 'ls -l' 1000 files with statahead: 7 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11080 miss_total: 20 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 29 seconds llite.lustre-ffff9a8e502b0800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11080 miss_total: 20 mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.batch_stats= snapshot_time: 1790926449.884810863 secs.nsecs start_time: 1790925980.977206745 secs.nsecs elapsed_time: 468.907604118 secs.nsecs subreqs per batch batches %% cum %% 1: 2 28 28 2: 0 0 28 4: 3 42 71 8: 0 0 71 16: 0 0 71 32: 0 0 71 64: 0 0 71 128: 0 0 71 256: 2 28 100 mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.batch_stats= snapshot_time: 1790926449.884877571 secs.nsecs start_time: 1790925980.977398335 secs.nsecs elapsed_time: 468.907479236 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 3 33 33 16: 2 22 55 32: 0 0 55 64: 2 22 77 128: 0 0 77 256: 2 22 100 - open/close 555 (time 1790926463.74 total 10.01 last 55.46) - open/close 1090 (time 1790926473.74 total 20.01 last 53.48) - open/close 1702 (time 1790926483.75 total 30.02 last 61.18) - open/close 2330 (time 1790926493.75 total 40.02 last 62.77) - open/close 2925 (time 1790926503.76 total 50.03 last 59.45) - open/close 3546 (time 1790926513.77 total 60.04 last 62.06) - open/close 4185 (time 1790926523.77 total 70.04 last 63.89) - open/close 4842 (time 1790926533.77 total 80.04 last 65.67) - open/close 5392 (time 1790926543.79 total 90.06 last 54.93) - open/close 5951 (time 1790926553.79 total 100.06 last 55.88) - open/close 6527 (time 1790926563.80 total 110.07 last 57.52) - open/close 7095 (time 1790926573.82 total 120.09 last 56.70) - open/close 7674 (time 1790926583.85 total 130.12 last 57.73) - open/close 8254 (time 1790926593.87 total 140.14 last 57.87) - open/close 8860 (time 1790926603.87 total 150.14 last 60.60) - open/close 9369 (time 1790926613.87 total 160.14 last 50.90) - open/close 9885 (time 1790926623.90 total 170.17 last 51.47) total: 10000 open/close in 172.22 seconds: 58.07 ops/second llite.lustre-ffff9a8e502b0800.statahead_batch_max=0 llite.lustre-ffff9a8e502b0800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.stats=clear mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m24.335s user 0m0.318s sys 0m14.978s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9a8e502b0800.statahead_batch_max=256 llite.lustre-ffff9a8e502b0800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.stats=clear mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m7.609s user 0m0.328s sys 0m14.659s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9a8e502b0800.batch_stats= snapshot_time: 1790926800.444859619 secs.nsecs start_time: 1790926720.585435864 secs.nsecs elapsed_time: 79.859423755 secs.nsecs subreqs per batch batches %% cum %% 1: 51 30 30 2: 24 14 45 4: 18 10 56 8: 8 4 60 16: 6 3 64 32: 17 10 74 64: 5 3 77 128: 1 0 78 256: 36 21 100 mdc.lustre-MDT0001-mdc-ffff9a8e502b0800.batch_stats= snapshot_time: 1790926800.447251815 secs.nsecs start_time: 1790926720.585509809 secs.nsecs elapsed_time: 79.861742006 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9a8e502b0800.statahead_batch_max=64 llite.lustre-ffff9a8e502b0800.statahead_max=128