== sanity test 123ad: Verify batching statahead works correctly ========================================================== 04:02:44 (1788768164) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9d87e0182800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d87e0182800.batch_stats=0 llite.lustre-ffff9d87e0182800.statahead_max=32 llite.lustre-ffff9d87e0182800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9d87e0182800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d87e0182800.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 1.78 seconds: 56.27 ops/second llite.lustre-ffff9d87e0182800.statahead_max=0 101 real 0m2.210s user 0m0.004s sys 0m0.322s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9d87e0182800.statahead_max=32 llite.lustre-ffff9d87e0182800.statahead_max=32 101 real 0m1.285s user 0m0.002s sys 0m0.186s 'ls -l' 100 files with statahead: 1 sec statahead total: 135 statahead wrong: 2 agl total: 129 list_total: 31 fname_total: 100 hit_total: 4223 miss_total: 757 swrong=2 ewrong=2 - open/close 870 (time 1788768190.56 total 10.03 last 86.74) total: 900 open/close in 10.62 seconds: 84.76 ops/second llite.lustre-ffff9d87e0182800.statahead_max=0 1001 real 0m23.055s user 0m0.032s sys 0m2.875s 'ls -l' 1000 files without statahead: 23 sec llite.lustre-ffff9d87e0182800.statahead_max=32 llite.lustre-ffff9d87e0182800.statahead_max=32 1001 real 0m8.971s user 0m0.033s sys 0m1.378s 'ls -l' 1000 files with statahead: 9 sec statahead total: 136 statahead wrong: 2 agl total: 130 list_total: 32 fname_total: 100 hit_total: 5222 miss_total: 758 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 23 seconds llite.lustre-ffff9d87e0182800.statahead_stats= statahead total: 136 statahead wrong: 2 agl total: 130 list_total: 32 fname_total: 100 hit_total: 5222 miss_total: 758 mdc.lustre-MDT0000-mdc-ffff9d87e0182800.batch_stats= snapshot_time: 1788768260.207103945 secs.nsecs start_time: 1788768168.494896734 secs.nsecs elapsed_time: 91.712207211 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 3 7 7 8: 4 10 17 16: 19 48 66 32: 13 33 100 mdc.lustre-MDT0001-mdc-ffff9d87e0182800.batch_stats= snapshot_time: 1788768260.207164418 secs.nsecs start_time: 1788768168.495011909 secs.nsecs elapsed_time: 91.712152509 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 1 2 5 4: 4 10 15 8: 0 0 15 16: 24 60 75 32: 10 25 100 - open/close 834 (time 1788768273.36 total 10.01 last 83.28) - open/close 1651 (time 1788768283.36 total 20.02 last 81.66) - open/close 2443 (time 1788768293.37 total 30.03 last 79.15) - open/close 3109 (time 1788768303.37 total 40.03 last 66.59) - open/close 3813 (time 1788768313.37 total 50.03 last 70.39) - open/close 4589 (time 1788768323.37 total 60.03 last 77.60) - open/close 5342 (time 1788768333.38 total 70.04 last 75.26) - open/close 6151 (time 1788768343.39 total 80.04 last 80.85) - open/close 7086 (time 1788768353.40 total 90.05 last 93.39) - open/close 7846 (time 1788768363.40 total 100.06 last 75.99) - open/close 8728 (time 1788768373.42 total 110.07 last 88.04) - open/close 9340 (time 1788768383.42 total 120.08 last 61.19) total: 10000 open/close in 128.27 seconds: 77.96 ops/second llite.lustre-ffff9d87e0182800.statahead_batch_max=0 llite.lustre-ffff9d87e0182800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d87e0182800.stats=clear mdc.lustre-MDT0001-mdc-ffff9d87e0182800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m10.667s user 0m0.324s sys 0m13.856s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9d87e0182800.statahead_batch_max=32 llite.lustre-ffff9d87e0182800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d87e0182800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9d87e0182800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9d87e0182800.stats=clear mdc.lustre-MDT0001-mdc-ffff9d87e0182800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m47.693s user 0m0.318s sys 0m13.548s batched statahead RPC= hit count=9976 mdc.lustre-MDT0000-mdc-ffff9d87e0182800.batch_stats= snapshot_time: 1788768594.192977379 secs.nsecs start_time: 1788768473.715181767 secs.nsecs elapsed_time: 120.477795612 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9d87e0182800.batch_stats= snapshot_time: 1788768594.193031572 secs.nsecs start_time: 1788768473.715308959 secs.nsecs elapsed_time: 120.477722613 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9d87e0182800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d87e0182800.batch_stats=0 llite.lustre-ffff9d87e0182800.statahead_max=2048 llite.lustre-ffff9d87e0182800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9d87e0182800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d87e0182800.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 2.05 seconds: 48.84 ops/second llite.lustre-ffff9d87e0182800.statahead_max=0 101 real 0m1.555s user 0m0.005s sys 0m0.280s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff9d87e0182800.statahead_max=2048 llite.lustre-ffff9d87e0182800.statahead_max=2048 101 real 0m0.865s user 0m0.005s sys 0m0.184s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10075 miss_total: 25 swrong=0 ewrong=0 total: 900 open/close in 8.89 seconds: 101.21 ops/second llite.lustre-ffff9d87e0182800.statahead_max=0 1001 real 0m20.649s user 0m0.032s sys 0m2.908s 'ls -l' 1000 files without statahead: 21 sec llite.lustre-ffff9d87e0182800.statahead_max=2048 llite.lustre-ffff9d87e0182800.statahead_max=2048 1001 real 0m7.460s user 0m0.039s sys 0m1.407s 'ls -l' 1000 files with statahead: 8 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11074 miss_total: 26 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 22 seconds llite.lustre-ffff9d87e0182800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11074 miss_total: 26 mdc.lustre-MDT0000-mdc-ffff9d87e0182800.batch_stats= snapshot_time: 1788768934.007022648 secs.nsecs start_time: 1788768596.368913821 secs.nsecs elapsed_time: 337.638108827 secs.nsecs subreqs per batch batches %% cum %% 1: 2 33 33 2: 1 16 50 4: 1 16 66 8: 0 0 66 16: 0 0 66 32: 0 0 66 64: 0 0 66 128: 0 0 66 256: 2 33 100 mdc.lustre-MDT0001-mdc-ffff9d87e0182800.batch_stats= snapshot_time: 1788768934.007120868 secs.nsecs start_time: 1788768596.369043902 secs.nsecs elapsed_time: 337.638076966 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 3 33 33 16: 1 11 44 32: 2 22 66 64: 1 11 77 128: 0 0 77 256: 2 22 100 - open/close 788 (time 1788768947.07 total 10.00 last 78.76) - open/close 1637 (time 1788768957.07 total 20.01 last 84.86) - open/close 2323 (time 1788768967.08 total 30.02 last 68.53) - open/close 3036 (time 1788768977.11 total 40.05 last 71.09) - open/close 3708 (time 1788768987.12 total 50.06 last 67.11) - open/close 4347 (time 1788768997.13 total 60.07 last 63.85) - open/close 5091 (time 1788769007.14 total 70.08 last 74.34) - open/close 5774 (time 1788769017.15 total 80.09 last 68.22) - open/close 6531 (time 1788769027.15 total 90.09 last 75.68) - open/close 7417 (time 1788769037.17 total 100.11 last 88.42) - open/close 8223 (time 1788769047.18 total 110.11 last 80.58) - open/close 9097 (time 1788769057.19 total 120.12 last 87.31) - open/close 9981 (time 1788769067.19 total 130.13 last 88.38) total: 10000 open/close in 130.29 seconds: 76.75 ops/second llite.lustre-ffff9d87e0182800.statahead_batch_max=0 llite.lustre-ffff9d87e0182800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d87e0182800.stats=clear mdc.lustre-MDT0001-mdc-ffff9d87e0182800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m15.477s user 0m0.322s sys 0m14.705s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9d87e0182800.statahead_batch_max=256 llite.lustre-ffff9d87e0182800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d87e0182800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9d87e0182800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9d87e0182800.stats=clear mdc.lustre-MDT0001-mdc-ffff9d87e0182800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m59.207s user 0m0.294s sys 0m14.563s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9d87e0182800.batch_stats= snapshot_time: 1788769218.164205992 secs.nsecs start_time: 1788769150.259039146 secs.nsecs elapsed_time: 67.905166846 secs.nsecs subreqs per batch batches %% cum %% 1: 126 44 44 2: 47 16 60 4: 27 9 70 8: 17 5 76 16: 10 3 79 32: 16 5 85 64: 6 2 87 128: 2 0 88 256: 34 11 100 mdc.lustre-MDT0001-mdc-ffff9d87e0182800.batch_stats= snapshot_time: 1788769218.164355613 secs.nsecs start_time: 1788769150.259111819 secs.nsecs elapsed_time: 67.905243794 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9d87e0182800.statahead_batch_max=64 llite.lustre-ffff9d87e0182800.statahead_max=128