== sanity test 123ad: Verify batching statahead works correctly ========================================================== 07:52:25 (1782733945) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9787d23d6000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9787d23d6000.batch_stats=0 llite.lustre-ffff9787d23d6000.statahead_max=32 llite.lustre-ffff9787d23d6000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9787d23d6000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9787d23d6000.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.61 seconds: 38.31 ops/second llite.lustre-ffff9787d23d6000.statahead_max=0 101 real 0m1.910s user 0m0.004s sys 0m0.330s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9787d23d6000.statahead_max=32 llite.lustre-ffff9787d23d6000.statahead_max=32 101 real 0m1.313s user 0m0.004s sys 0m0.204s 'ls -l' 100 files with statahead: 1 sec statahead total: 139 statahead wrong: 2 agl total: 133 list_total: 31 fname_total: 107 hit_total: 3833 miss_total: 1571 swrong=2 ewrong=2 - open/close 789 (time 1782733973.49 total 10.00 last 78.88) total: 900 open/close in 11.58 seconds: 77.71 ops/second llite.lustre-ffff9787d23d6000.statahead_max=0 1001 real 0m21.511s user 0m0.029s sys 0m3.110s 'ls -l' 1000 files without statahead: 21 sec llite.lustre-ffff9787d23d6000.statahead_max=32 llite.lustre-ffff9787d23d6000.statahead_max=32 1001 real 0m9.479s user 0m0.023s sys 0m1.450s 'ls -l' 1000 files with statahead: 9 sec statahead total: 140 statahead wrong: 2 agl total: 134 list_total: 32 fname_total: 107 hit_total: 4832 miss_total: 1572 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 23 seconds llite.lustre-ffff9787d23d6000.statahead_stats= statahead total: 140 statahead wrong: 2 agl total: 134 list_total: 32 fname_total: 107 hit_total: 4832 miss_total: 1572 mdc.lustre-MDT0000-mdc-ffff9787d23d6000.batch_stats= snapshot_time: 1782734043.263378614 secs.nsecs start_time: 1782733951.318225652 secs.nsecs elapsed_time: 91.945152962 secs.nsecs subreqs per batch batches %% cum %% 1: 2 5 5 2: 0 0 5 4: 4 10 16 8: 0 0 16 16: 26 70 86 32: 5 13 100 mdc.lustre-MDT0001-mdc-ffff9787d23d6000.batch_stats= snapshot_time: 1782734043.263444741 secs.nsecs start_time: 1782733951.318303106 secs.nsecs elapsed_time: 91.945141635 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 1 2 5 8: 3 7 12 16: 7 17 30 32: 27 69 100 - open/close 732 (time 1782734056.02 total 10.01 last 73.10) - open/close 1392 (time 1782734066.03 total 20.03 last 65.91) - open/close 2119 (time 1782734076.04 total 30.03 last 72.65) - open/close 2938 (time 1782734086.04 total 40.03 last 81.89) - open/close 3736 (time 1782734096.04 total 50.04 last 79.75) - open/close 4375 (time 1782734106.05 total 60.05 last 63.85) - open/close 5052 (time 1782734116.05 total 70.05 last 67.68) - open/close 5771 (time 1782734126.06 total 80.06 last 71.88) - open/close 6655 (time 1782734136.06 total 90.06 last 88.39) - open/close 7252 (time 1782734146.07 total 100.07 last 59.64) - open/close 8024 (time 1782734156.08 total 110.07 last 77.15) - open/close 8781 (time 1782734166.08 total 120.07 last 75.70) - open/close 9700 (time 1782734176.08 total 130.07 last 91.90) total: 10000 open/close in 133.78 seconds: 74.75 ops/second llite.lustre-ffff9787d23d6000.statahead_batch_max=0 llite.lustre-ffff9787d23d6000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9787d23d6000.stats=clear mdc.lustre-MDT0001-mdc-ffff9787d23d6000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m0.632s user 0m0.279s sys 0m15.341s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9787d23d6000.statahead_batch_max=32 llite.lustre-ffff9787d23d6000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9787d23d6000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9787d23d6000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9787d23d6000.stats=clear mdc.lustre-MDT0001-mdc-ffff9787d23d6000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m55.914s user 0m0.314s sys 0m14.960s batched statahead RPC= hit count=9979 mdc.lustre-MDT0000-mdc-ffff9787d23d6000.batch_stats= snapshot_time: 1782734382.678344572 secs.nsecs start_time: 1782734252.745705972 secs.nsecs elapsed_time: 129.932638600 secs.nsecs subreqs per batch batches %% cum %% 1: 1 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9787d23d6000.batch_stats= snapshot_time: 1782734382.678409253 secs.nsecs start_time: 1782734252.745803706 secs.nsecs elapsed_time: 129.932605547 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9787d23d6000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9787d23d6000.batch_stats=0 llite.lustre-ffff9787d23d6000.statahead_max=2048 llite.lustre-ffff9787d23d6000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9787d23d6000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9787d23d6000.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.99 seconds: 33.47 ops/second llite.lustre-ffff9787d23d6000.statahead_max=0 101 real 0m2.820s user 0m0.008s sys 0m0.385s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9787d23d6000.statahead_max=2048 llite.lustre-ffff9787d23d6000.statahead_max=2048 101 real 0m0.868s user 0m0.005s sys 0m0.181s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10078 miss_total: 22 swrong=0 ewrong=0 - open/close 802 (time 1782734702.13 total 10.00 last 80.16) total: 900 open/close in 11.26 seconds: 79.95 ops/second llite.lustre-ffff9787d23d6000.statahead_max=0 1001 real 0m22.939s user 0m0.032s sys 0m3.080s 'ls -l' 1000 files without statahead: 22 sec llite.lustre-ffff9787d23d6000.statahead_max=2048 llite.lustre-ffff9787d23d6000.statahead_max=2048 1001 real 0m10.705s user 0m0.040s sys 0m1.507s 'ls -l' 1000 files with statahead: 10 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11077 miss_total: 23 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 22 seconds llite.lustre-ffff9787d23d6000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11077 miss_total: 23 mdc.lustre-MDT0000-mdc-ffff9787d23d6000.batch_stats= snapshot_time: 1782734773.496919711 secs.nsecs start_time: 1782734386.250109027 secs.nsecs elapsed_time: 387.246810684 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 11 11 8: 4 44 55 16: 1 11 66 32: 1 11 77 64: 0 0 77 128: 0 0 77 256: 2 22 100 mdc.lustre-MDT0001-mdc-ffff9787d23d6000.batch_stats= snapshot_time: 1782734773.496982272 secs.nsecs start_time: 1782734386.250268597 secs.nsecs elapsed_time: 387.246713675 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 1 11 11 4: 2 22 33 8: 2 22 55 16: 1 11 66 32: 0 0 66 64: 1 11 77 128: 0 0 77 256: 2 22 100 - open/close 709 (time 1782734786.35 total 10.01 last 70.86) - open/close 1538 (time 1782734796.36 total 20.02 last 82.80) - open/close 2399 (time 1782734806.37 total 30.03 last 85.96) - open/close 3289 (time 1782734816.38 total 40.04 last 88.95) - open/close 4054 (time 1782734826.38 total 50.04 last 76.47) - open/close 4903 (time 1782734836.39 total 60.05 last 84.85) - open/close 5740 (time 1782734846.40 total 70.06 last 83.61) - open/close 6277 (time 1782734856.41 total 80.07 last 53.66) - open/close 6894 (time 1782734866.42 total 90.08 last 61.62) - open/close 7615 (time 1782734876.42 total 100.08 last 72.09) - open/close 8358 (time 1782734886.43 total 110.09 last 74.23) - open/close 9128 (time 1782734896.44 total 120.10 last 76.95) - open/close 9791 (time 1782734906.45 total 130.11 last 66.24) total: 10000 open/close in 133.46 seconds: 74.93 ops/second llite.lustre-ffff9787d23d6000.statahead_batch_max=0 llite.lustre-ffff9787d23d6000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9787d23d6000.stats=clear mdc.lustre-MDT0001-mdc-ffff9787d23d6000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m11.552s user 0m0.338s sys 0m15.788s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9787d23d6000.statahead_batch_max=256 llite.lustre-ffff9787d23d6000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9787d23d6000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9787d23d6000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9787d23d6000.stats=clear mdc.lustre-MDT0001-mdc-ffff9787d23d6000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m58.154s user 0m0.313s sys 0m15.531s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9787d23d6000.batch_stats= snapshot_time: 1782735070.369672536 secs.nsecs start_time: 1782734995.096116100 secs.nsecs elapsed_time: 75.273556436 secs.nsecs subreqs per batch batches %% cum %% 1: 202 53 53 2: 57 15 68 4: 35 9 77 8: 19 5 82 16: 8 2 84 32: 17 4 88 64: 8 2 91 128: 0 0 91 256: 34 8 100 mdc.lustre-MDT0001-mdc-ffff9787d23d6000.batch_stats= snapshot_time: 1782735070.369728466 secs.nsecs start_time: 1782734995.096227237 secs.nsecs elapsed_time: 75.273501229 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9787d23d6000.statahead_batch_max=64 llite.lustre-ffff9787d23d6000.statahead_max=128