== sanity test 123ad: Verify batching statahead works correctly ========================================================== 02:31:33 (1788935493) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.batch_stats=0 llite.lustre-ffff9d6b61ebe000.statahead_max=32 llite.lustre-ffff9d6b61ebe000.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 3.12 seconds: 32.07 ops/second llite.lustre-ffff9d6b61ebe000.statahead_max=0 101 real 0m3.083s user 0m0.005s sys 0m0.338s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff9d6b61ebe000.statahead_max=32 llite.lustre-ffff9d6b61ebe000.statahead_max=32 101 real 0m1.501s user 0m0.005s sys 0m0.169s 'ls -l' 100 files with statahead: 1 sec statahead total: 118 statahead wrong: 2 agl total: 112 list_total: 34 fname_total: 72 hit_total: 4431 miss_total: 1283 swrong=2 ewrong=2 - open/close 611 (time 1788935526.03 total 10.01 last 61.06) total: 900 open/close in 15.31 seconds: 58.77 ops/second llite.lustre-ffff9d6b61ebe000.statahead_max=0 1001 real 0m31.005s user 0m0.036s sys 0m3.055s 'ls -l' 1000 files without statahead: 31 sec llite.lustre-ffff9d6b61ebe000.statahead_max=32 llite.lustre-ffff9d6b61ebe000.statahead_max=32 1001 real 0m8.042s user 0m0.035s sys 0m1.429s 'ls -l' 1000 files with statahead: 9 sec statahead total: 119 statahead wrong: 2 agl total: 113 list_total: 35 fname_total: 72 hit_total: 5428 miss_total: 1286 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 24 seconds llite.lustre-ffff9d6b61ebe000.statahead_stats= statahead total: 119 statahead wrong: 2 agl total: 113 list_total: 35 fname_total: 72 hit_total: 5428 miss_total: 1286 mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.batch_stats= snapshot_time: 1788935607.992456141 secs.nsecs start_time: 1788935499.502881915 secs.nsecs elapsed_time: 108.489574226 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 2 5 8 4: 1 2 10 8: 4 10 21 16: 21 56 78 32: 8 21 100 mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.batch_stats= snapshot_time: 1788935607.994239554 secs.nsecs start_time: 1788935499.503031348 secs.nsecs elapsed_time: 108.491208206 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 1 2 5 4: 2 5 10 8: 1 2 12 16: 14 35 47 32: 21 52 100 - open/close 703 (time 1788935620.60 total 10.03 last 70.06) - open/close 1384 (time 1788935630.61 total 20.04 last 68.07) - open/close 2104 (time 1788935640.61 total 30.05 last 71.95) - open/close 2820 (time 1788935650.62 total 40.05 last 71.57) - open/close 3573 (time 1788935660.64 total 50.07 last 75.16) - open/close 4332 (time 1788935670.65 total 60.08 last 75.84) - open/close 5044 (time 1788935680.65 total 70.08 last 71.18) - open/close 5745 (time 1788935690.66 total 80.09 last 70.02) - open/close 6421 (time 1788935700.68 total 90.11 last 67.48) - open/close 7141 (time 1788935710.68 total 100.12 last 71.96) - open/close 7818 (time 1788935720.68 total 110.12 last 67.70) - open/close 8513 (time 1788935730.70 total 120.13 last 69.41) - open/close 9199 (time 1788935740.71 total 130.14 last 68.54) - open/close 9931 (time 1788935750.71 total 140.15 last 73.14) total: 10000 open/close in 140.87 seconds: 70.99 ops/second llite.lustre-ffff9d6b61ebe000.statahead_batch_max=0 llite.lustre-ffff9d6b61ebe000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.stats=clear mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m4.112s user 0m0.316s sys 0m15.023s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9d6b61ebe000.statahead_batch_max=32 llite.lustre-ffff9d6b61ebe000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.stats=clear mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.stats=clear ls -l files in d123ad.sanity: 10001 real 1m44.380s user 0m0.306s sys 0m14.430s batched statahead RPC= hit count=9976 mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.batch_stats= snapshot_time: 1788935939.569590237 secs.nsecs start_time: 1788935824.626714336 secs.nsecs elapsed_time: 114.942875901 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 0 0 8: 1 0 0 16: 0 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.batch_stats= snapshot_time: 1788935939.569679407 secs.nsecs start_time: 1788935824.626897421 secs.nsecs elapsed_time: 114.942781986 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.batch_stats=0 llite.lustre-ffff9d6b61ebe000.statahead_max=2048 llite.lustre-ffff9d6b61ebe000.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.17 seconds: 31.59 ops/second llite.lustre-ffff9d6b61ebe000.statahead_max=0 101 real 0m2.521s user 0m0.003s sys 0m0.295s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff9d6b61ebe000.statahead_max=2048 llite.lustre-ffff9d6b61ebe000.statahead_max=2048 101 real 0m0.844s user 0m0.007s sys 0m0.189s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10075 miss_total: 25 swrong=0 ewrong=0 - open/close 740 (time 1788936276.63 total 10.01 last 73.89) total: 900 open/close in 12.17 seconds: 73.97 ops/second llite.lustre-ffff9d6b61ebe000.statahead_max=0 1001 real 0m21.310s user 0m0.038s sys 0m2.898s 'ls -l' 1000 files without statahead: 22 sec llite.lustre-ffff9d6b61ebe000.statahead_max=2048 llite.lustre-ffff9d6b61ebe000.statahead_max=2048 1001 real 0m6.782s user 0m0.023s sys 0m1.279s 'ls -l' 1000 files with statahead: 7 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11074 miss_total: 26 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 23 seconds llite.lustre-ffff9d6b61ebe000.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11074 miss_total: 26 mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.batch_stats= snapshot_time: 1788936343.479312768 secs.nsecs start_time: 1788935942.172321169 secs.nsecs elapsed_time: 401.306991599 secs.nsecs subreqs per batch batches %% cum %% 1: 1 12 12 2: 2 25 37 4: 1 12 50 8: 2 25 75 16: 0 0 75 32: 0 0 75 64: 0 0 75 128: 0 0 75 256: 2 25 100 mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.batch_stats= snapshot_time: 1788936343.479369114 secs.nsecs start_time: 1788935942.172513959 secs.nsecs elapsed_time: 401.306855155 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 20 20 8: 1 10 30 16: 3 30 60 32: 1 10 70 64: 1 10 80 128: 0 0 80 256: 2 20 100 - open/close 662 (time 1788936356.55 total 10.02 last 66.10) - open/close 1425 (time 1788936366.55 total 20.02 last 76.28) - open/close 2130 (time 1788936376.55 total 30.02 last 70.48) - open/close 2821 (time 1788936386.55 total 40.02 last 69.09) - open/close 3491 (time 1788936396.56 total 50.03 last 66.97) - open/close 4089 (time 1788936406.56 total 60.03 last 59.76) - open/close 4577 (time 1788936416.57 total 70.04 last 48.74) - open/close 5173 (time 1788936426.59 total 80.06 last 59.53) - open/close 5770 (time 1788936436.60 total 90.07 last 59.62) - open/close 6492 (time 1788936446.60 total 100.07 last 72.18) - open/close 7188 (time 1788936456.61 total 110.08 last 69.52) - open/close 8038 (time 1788936466.62 total 120.09 last 84.91) - open/close 8806 (time 1788936476.63 total 130.10 last 76.76) - open/close 9554 (time 1788936486.64 total 140.11 last 74.74) total: 10000 open/close in 146.84 seconds: 68.10 ops/second llite.lustre-ffff9d6b61ebe000.statahead_batch_max=0 llite.lustre-ffff9d6b61ebe000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.stats=clear mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m56.563s user 0m0.284s sys 0m15.615s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff9d6b61ebe000.statahead_batch_max=256 llite.lustre-ffff9d6b61ebe000.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.stats=clear mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.stats=clear ls -l files in d123ad.sanity: 10001 real 0m50.940s user 0m0.306s sys 0m15.324s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff9d6b61ebe000.batch_stats= snapshot_time: 1788936620.794922210 secs.nsecs start_time: 1788936558.866388139 secs.nsecs elapsed_time: 61.928534071 secs.nsecs subreqs per batch batches %% cum %% 1: 94 45 45 2: 19 9 54 4: 16 7 62 8: 12 5 67 16: 6 2 70 32: 22 10 81 64: 4 1 83 128: 0 0 83 256: 35 16 100 mdc.lustre-MDT0001-mdc-ffff9d6b61ebe000.batch_stats= snapshot_time: 1788936620.794981027 secs.nsecs start_time: 1788936558.866589505 secs.nsecs elapsed_time: 61.928391522 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff9d6b61ebe000.statahead_batch_max=64 llite.lustre-ffff9d6b61ebe000.statahead_max=128