== sanity test 123ad: Verify batching statahead works correctly ========================================================== 09:06:09 (1790946369) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff8dc82d687800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8dc82d687800.batch_stats=0 llite.lustre-ffff8dc82d687800.statahead_max=32 llite.lustre-ffff8dc82d687800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff8dc82d687800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8dc82d687800.batch_stats=0 striped dir -i1 -c2 -H all_char /mnt/lustre/d123ad.sanity total: 100 open/close in 1.91 seconds: 52.46 ops/second llite.lustre-ffff8dc82d687800.statahead_max=0 101 real 0m1.872s user 0m0.006s sys 0m0.303s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff8dc82d687800.statahead_max=32 llite.lustre-ffff8dc82d687800.statahead_max=32 101 real 0m0.966s user 0m0.005s sys 0m0.176s 'ls -l' 100 files with statahead: 1 sec statahead total: 143 statahead wrong: 2 agl total: 137 list_total: 32 fname_total: 89 hit_total: 3707 miss_total: 1621 swrong=2 ewrong=2 - open/close 804 (time 1790946396.84 total 10.00 last 80.36) total: 900 open/close in 11.23 seconds: 80.14 ops/second llite.lustre-ffff8dc82d687800.statahead_max=0 1001 real 0m19.628s user 0m0.032s sys 0m2.871s 'ls -l' 1000 files without statahead: 20 sec llite.lustre-ffff8dc82d687800.statahead_max=32 llite.lustre-ffff8dc82d687800.statahead_max=32 1001 real 0m9.300s user 0m0.020s sys 0m1.358s 'ls -l' 1000 files with statahead: 9 sec statahead total: 144 statahead wrong: 2 agl total: 138 list_total: 33 fname_total: 89 hit_total: 4705 miss_total: 1623 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 21 seconds llite.lustre-ffff8dc82d687800.statahead_stats= statahead total: 144 statahead wrong: 2 agl total: 138 list_total: 33 fname_total: 89 hit_total: 4705 miss_total: 1623 mdc.lustre-MDT0000-mdc-ffff8dc82d687800.batch_stats= snapshot_time: 1790946459.694479897 secs.nsecs start_time: 1790946375.437569279 secs.nsecs elapsed_time: 84.256910618 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 2 5 5 4: 1 2 7 8: 4 10 17 16: 19 48 66 32: 13 33 100 mdc.lustre-MDT0001-mdc-ffff8dc82d687800.batch_stats= snapshot_time: 1790946459.696116993 secs.nsecs start_time: 1790946375.437964708 secs.nsecs elapsed_time: 84.258152285 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 3 7 10 8: 3 7 17 16: 19 47 65 32: 14 35 100 - open/close 939 (time 1790946472.05 total 10.00 last 93.88) - open/close 1884 (time 1790946482.05 total 20.01 last 94.45) - open/close 2938 (time 1790946492.06 total 30.01 last 105.38) - open/close 3846 (time 1790946502.06 total 40.01 last 90.79) - open/close 4674 (time 1790946512.08 total 50.04 last 82.58) - open/close 5649 (time 1790946522.11 total 60.06 last 97.28) - open/close 6407 (time 1790946532.12 total 70.08 last 75.66) - open/close 7375 (time 1790946542.13 total 80.09 last 96.73) - open/close 8317 (time 1790946552.14 total 90.09 last 94.13) - open/close 9249 (time 1790946562.14 total 100.10 last 93.18) total: 10000 open/close in 107.94 seconds: 92.64 ops/second llite.lustre-ffff8dc82d687800.statahead_batch_max=0 llite.lustre-ffff8dc82d687800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8dc82d687800.stats=clear mdc.lustre-MDT0001-mdc-ffff8dc82d687800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m52.978s user 0m0.296s sys 0m13.825s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8dc82d687800.statahead_batch_max=32 llite.lustre-ffff8dc82d687800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8dc82d687800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8dc82d687800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8dc82d687800.stats=clear mdc.lustre-MDT0001-mdc-ffff8dc82d687800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m33.772s user 0m0.273s sys 0m13.027s batched statahead RPC= hit count=9983 mdc.lustre-MDT0000-mdc-ffff8dc82d687800.batch_stats= snapshot_time: 1790946735.974539361 secs.nsecs start_time: 1790946631.498285204 secs.nsecs elapsed_time: 104.476254157 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff8dc82d687800.batch_stats= snapshot_time: 1790946735.974775497 secs.nsecs start_time: 1790946631.498358016 secs.nsecs elapsed_time: 104.476417481 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff8dc82d687800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8dc82d687800.batch_stats=0 llite.lustre-ffff8dc82d687800.statahead_max=2048 llite.lustre-ffff8dc82d687800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff8dc82d687800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff8dc82d687800.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 2.62 seconds: 38.15 ops/second llite.lustre-ffff8dc82d687800.statahead_max=0 101 real 0m2.192s user 0m0.004s sys 0m0.310s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff8dc82d687800.statahead_max=2048 llite.lustre-ffff8dc82d687800.statahead_max=2048 101 real 0m0.845s user 0m0.003s sys 0m0.176s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10082 miss_total: 18 swrong=0 ewrong=0 - open/close 869 (time 1790947005.87 total 10.01 last 86.84) total: 900 open/close in 10.29 seconds: 87.48 ops/second llite.lustre-ffff8dc82d687800.statahead_max=0 1001 real 0m21.259s user 0m0.027s sys 0m2.757s 'ls -l' 1000 files without statahead: 21 sec llite.lustre-ffff8dc82d687800.statahead_max=2048 llite.lustre-ffff8dc82d687800.statahead_max=2048 1001 real 0m6.185s user 0m0.024s sys 0m1.233s 'ls -l' 1000 files with statahead: 6 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11081 miss_total: 19 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 19 seconds llite.lustre-ffff8dc82d687800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11081 miss_total: 19 mdc.lustre-MDT0000-mdc-ffff8dc82d687800.batch_stats= snapshot_time: 1790947064.493247760 secs.nsecs start_time: 1790946738.038994659 secs.nsecs elapsed_time: 326.454253101 secs.nsecs subreqs per batch batches %% cum %% 1: 2 22 22 2: 1 11 33 4: 2 22 55 8: 1 11 66 16: 0 0 66 32: 0 0 66 64: 1 11 77 128: 0 0 77 256: 2 22 100 mdc.lustre-MDT0001-mdc-ffff8dc82d687800.batch_stats= snapshot_time: 1790947064.493333837 secs.nsecs start_time: 1790946738.039141663 secs.nsecs elapsed_time: 326.454192174 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 3 27 27 16: 3 27 54 32: 2 18 72 64: 1 9 81 128: 0 0 81 256: 2 18 100 - open/close 928 (time 1790947076.99 total 10.01 last 92.72) - open/close 1768 (time 1790947087.00 total 20.01 last 83.95) - open/close 2717 (time 1790947097.01 total 30.03 last 94.75) - open/close 3659 (time 1790947107.02 total 40.04 last 94.09) - open/close 4632 (time 1790947117.03 total 50.04 last 97.29) - open/close 5546 (time 1790947127.03 total 60.04 last 91.38) - open/close 6493 (time 1790947137.04 total 70.05 last 94.62) - open/close 7434 (time 1790947147.05 total 80.06 last 93.99) - open/close 8313 (time 1790947157.05 total 90.07 last 87.89) - open/close 9255 (time 1790947167.06 total 100.08 last 94.09) total: 10000 open/close in 107.64 seconds: 92.90 ops/second llite.lustre-ffff8dc82d687800.statahead_batch_max=0 llite.lustre-ffff8dc82d687800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8dc82d687800.stats=clear mdc.lustre-MDT0001-mdc-ffff8dc82d687800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m56.598s user 0m0.324s sys 0m14.366s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff8dc82d687800.statahead_batch_max=256 llite.lustre-ffff8dc82d687800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff8dc82d687800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff8dc82d687800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff8dc82d687800.stats=clear mdc.lustre-MDT0001-mdc-ffff8dc82d687800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m46.627s user 0m0.280s sys 0m14.287s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff8dc82d687800.batch_stats= snapshot_time: 1790947297.658249595 secs.nsecs start_time: 1790947239.166109232 secs.nsecs elapsed_time: 58.492140363 secs.nsecs subreqs per batch batches %% cum %% 1: 111 41 41 2: 42 15 57 4: 22 8 65 8: 27 10 75 16: 8 2 78 32: 17 6 84 64: 6 2 86 128: 0 0 86 256: 35 13 100 mdc.lustre-MDT0001-mdc-ffff8dc82d687800.batch_stats= snapshot_time: 1790947297.658315490 secs.nsecs start_time: 1790947239.166260720 secs.nsecs elapsed_time: 58.492054770 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff8dc82d687800.statahead_batch_max=64 llite.lustre-ffff8dc82d687800.statahead_max=128