== sanity test 123ad: Verify batching statahead works correctly ========================================================== 18:02:28 (1788818548) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffffa09890daf800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffffa09890daf800.batch_stats=0 llite.lustre-ffffa09890daf800.statahead_max=32 llite.lustre-ffffa09890daf800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffffa09890daf800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffffa09890daf800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.54 seconds: 39.34 ops/second llite.lustre-ffffa09890daf800.statahead_max=0 101 real 0m2.521s user 0m0.006s sys 0m0.337s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffffa09890daf800.statahead_max=32 llite.lustre-ffffa09890daf800.statahead_max=32 101 real 0m1.185s user 0m0.004s sys 0m0.203s 'ls -l' 100 files with statahead: 1 sec statahead total: 122 statahead wrong: 2 agl total: 117 list_total: 31 fname_total: 89 hit_total: 3999 miss_total: 1611 swrong=2 ewrong=2 - open/close 736 (time 1788818576.72 total 10.01 last 73.52) total: 900 open/close in 12.36 seconds: 72.79 ops/second llite.lustre-ffffa09890daf800.statahead_max=0 1001 real 0m23.770s user 0m0.036s sys 0m3.230s 'ls -l' 1000 files without statahead: 23 sec llite.lustre-ffffa09890daf800.statahead_max=32 llite.lustre-ffffa09890daf800.statahead_max=32 1001 real 0m10.935s user 0m0.037s sys 0m1.608s 'ls -l' 1000 files with statahead: 11 sec statahead total: 123 statahead wrong: 2 agl total: 118 list_total: 32 fname_total: 89 hit_total: 4996 miss_total: 1614 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 23 seconds llite.lustre-ffffa09890daf800.statahead_stats= statahead total: 123 statahead wrong: 2 agl total: 118 list_total: 32 fname_total: 89 hit_total: 4996 miss_total: 1614 mdc.lustre-MDT0000-mdc-ffffa09890daf800.batch_stats= snapshot_time: 1788818651.232041448 secs.nsecs start_time: 1788818553.492532081 secs.nsecs elapsed_time: 97.739509367 secs.nsecs subreqs per batch batches %% cum %% 1: 2 5 5 2: 1 2 8 4: 2 5 13 8: 2 5 18 16: 25 67 86 32: 5 13 100 mdc.lustre-MDT0001-mdc-ffffa09890daf800.batch_stats= snapshot_time: 1788818651.232119933 secs.nsecs start_time: 1788818553.492618116 secs.nsecs elapsed_time: 97.739501817 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 1 2 2 4: 0 0 2 8: 3 7 10 16: 12 31 42 32: 22 57 100 - open/close 645 (time 1788818663.95 total 10.00 last 64.47) - open/close 1248 (time 1788818673.96 total 20.02 last 60.22) - open/close 1908 (time 1788818683.97 total 30.03 last 65.92) - open/close 2545 (time 1788818693.97 total 40.03 last 63.68) - open/close 3264 (time 1788818703.98 total 50.04 last 71.87) - open/close 4065 (time 1788818713.99 total 60.05 last 80.04) - open/close 4793 (time 1788818723.99 total 70.05 last 72.78) - open/close 5568 (time 1788818733.99 total 80.05 last 77.48) - open/close 6264 (time 1788818744.00 total 90.06 last 69.55) - open/close 6974 (time 1788818754.00 total 100.06 last 70.99) - open/close 7674 (time 1788818764.00 total 110.06 last 69.98) - open/close 8397 (time 1788818774.02 total 120.08 last 72.18) - open/close 9064 (time 1788818784.05 total 130.11 last 66.51) - open/close 9634 (time 1788818794.05 total 140.11 last 56.96) total: 10000 open/close in 146.07 seconds: 68.46 ops/second llite.lustre-ffffa09890daf800.statahead_batch_max=0 llite.lustre-ffffa09890daf800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffffa09890daf800.stats=clear mdc.lustre-MDT0001-mdc-ffffa09890daf800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m57.679s user 0m0.320s sys 0m16.327s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffffa09890daf800.statahead_batch_max=32 llite.lustre-ffffa09890daf800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffffa09890daf800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffffa09890daf800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffffa09890daf800.stats=clear mdc.lustre-MDT0001-mdc-ffffa09890daf800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m47.479s user 0m0.310s sys 0m14.328s batched statahead RPC= hit count=9986 mdc.lustre-MDT0000-mdc-ffffa09890daf800.batch_stats= snapshot_time: 1788818983.965344391 secs.nsecs start_time: 1788818866.647595769 secs.nsecs elapsed_time: 117.317748622 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffffa09890daf800.batch_stats= snapshot_time: 1788818983.965427273 secs.nsecs start_time: 1788818866.648064667 secs.nsecs elapsed_time: 117.317362606 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffffa09890daf800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffffa09890daf800.batch_stats=0 llite.lustre-ffffa09890daf800.statahead_max=2048 llite.lustre-ffffa09890daf800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffffa09890daf800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffffa09890daf800.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.03 seconds: 49.28 ops/second llite.lustre-ffffa09890daf800.statahead_max=0 101 real 0m2.727s user 0m0.006s sys 0m0.307s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffffa09890daf800.statahead_max=2048 llite.lustre-ffffa09890daf800.statahead_max=2048 101 real 0m0.759s user 0m0.008s sys 0m0.183s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10085 miss_total: 15 swrong=0 ewrong=0 - open/close 871 (time 1788819267.71 total 10.01 last 87.00) total: 900 open/close in 10.49 seconds: 85.83 ops/second llite.lustre-ffffa09890daf800.statahead_max=0 1001 real 0m25.086s user 0m0.036s sys 0m2.899s 'ls -l' 1000 files without statahead: 26 sec llite.lustre-ffffa09890daf800.statahead_max=2048 llite.lustre-ffffa09890daf800.statahead_max=2048 1001 real 0m7.506s user 0m0.031s sys 0m1.397s 'ls -l' 1000 files with statahead: 8 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11084 miss_total: 16 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 23 seconds llite.lustre-ffffa09890daf800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11084 miss_total: 16 mdc.lustre-MDT0000-mdc-ffffa09890daf800.batch_stats= snapshot_time: 1788819336.871870757 secs.nsecs start_time: 1788818986.657513214 secs.nsecs elapsed_time: 350.214357543 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 1 11 11 8: 4 44 55 16: 0 0 55 32: 1 11 66 64: 1 11 77 128: 0 0 77 256: 2 22 100 mdc.lustre-MDT0001-mdc-ffffa09890daf800.batch_stats= snapshot_time: 1788819336.871955138 secs.nsecs start_time: 1788818986.657699525 secs.nsecs elapsed_time: 350.214255613 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 1 12 12 4: 1 12 25 8: 1 12 37 16: 2 25 62 32: 0 0 62 64: 1 12 75 128: 0 0 75 256: 2 25 100 - open/close 729 (time 1788819350.23 total 10.00 last 72.87) - open/close 1523 (time 1788819360.24 total 20.01 last 79.33) - open/close 2313 (time 1788819370.25 total 30.02 last 78.94) - open/close 3055 (time 1788819380.25 total 40.02 last 74.19) - open/close 3669 (time 1788819390.26 total 50.03 last 61.36) - open/close 4318 (time 1788819400.26 total 60.03 last 64.88) - open/close 5025 (time 1788819410.27 total 70.04 last 70.63) - open/close 5719 (time 1788819420.27 total 80.05 last 69.36) - open/close 6551 (time 1788819430.28 total 90.05 last 83.15) - open/close 7318 (time 1788819440.30 total 100.07 last 76.56) - open/close 8125 (time 1788819450.31 total 110.08 last 80.63) - open/close 8837 (time 1788819460.31 total 120.08 last 71.19) - open/close 9493 (time 1788819470.32 total 130.09 last 65.55) total: 10000 open/close in 138.28 seconds: 72.32 ops/second llite.lustre-ffffa09890daf800.statahead_batch_max=0 llite.lustre-ffffa09890daf800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffffa09890daf800.stats=clear mdc.lustre-MDT0001-mdc-ffffa09890daf800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m17.359s user 0m0.354s sys 0m15.082s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffffa09890daf800.statahead_batch_max=256 llite.lustre-ffffa09890daf800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffffa09890daf800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffffa09890daf800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffffa09890daf800.stats=clear mdc.lustre-MDT0001-mdc-ffffa09890daf800.stats=clear ls -l files in d123ad.sanity: 10001 real 0m51.535s user 0m0.319s sys 0m14.841s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffffa09890daf800.batch_stats= snapshot_time: 1788819626.108802537 secs.nsecs start_time: 1788819564.350907139 secs.nsecs elapsed_time: 61.757895398 secs.nsecs subreqs per batch batches %% cum %% 1: 92 38 38 2: 42 17 56 4: 23 9 66 8: 11 4 71 16: 5 2 73 32: 18 7 80 64: 8 3 84 128: 3 1 85 256: 34 14 100 mdc.lustre-MDT0001-mdc-ffffa09890daf800.batch_stats= snapshot_time: 1788819626.108873230 secs.nsecs start_time: 1788819564.351098501 secs.nsecs elapsed_time: 61.757774729 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffffa09890daf800.statahead_batch_max=64 llite.lustre-ffffa09890daf800.statahead_max=128