== sanity test 123ad: Verify batching statahead works correctly ========================================================== 03:31:53 (1785396713) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.batch_stats=0 llite.lustre-ffff955c8cf8a800.statahead_max=32 llite.lustre-ffff955c8cf8a800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.batch_stats=0 striped dir -i1 -c2 -H crush2 /mnt/lustre/d123ad.sanity total: 100 open/close in 2.34 seconds: 42.65 ops/second llite.lustre-ffff955c8cf8a800.statahead_max=0 101 real 0m1.914s user 0m0.005s sys 0m0.315s 'ls -l' 100 files without statahead: 2 sec llite.lustre-ffff955c8cf8a800.statahead_max=32 llite.lustre-ffff955c8cf8a800.statahead_max=32 101 real 0m1.739s user 0m0.005s sys 0m0.175s 'ls -l' 100 files with statahead: 1 sec statahead total: 163 statahead wrong: 2 agl total: 155 list_total: 32 fname_total: 116 hit_total: 3634 miss_total: 1575 swrong=2 ewrong=2 - open/close 543 (time 1785396743.63 total 10.00 last 54.28) total: 900 open/close in 15.22 seconds: 59.15 ops/second llite.lustre-ffff955c8cf8a800.statahead_max=0 1001 real 0m37.747s user 0m0.039s sys 0m3.194s 'ls -l' 1000 files without statahead: 38 sec llite.lustre-ffff955c8cf8a800.statahead_max=32 llite.lustre-ffff955c8cf8a800.statahead_max=32 1001 real 0m14.467s user 0m0.027s sys 0m1.539s 'ls -l' 1000 files with statahead: 15 sec statahead total: 164 statahead wrong: 2 agl total: 156 list_total: 33 fname_total: 116 hit_total: 4633 miss_total: 1576 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 32 seconds llite.lustre-ffff955c8cf8a800.statahead_stats= statahead total: 164 statahead wrong: 2 agl total: 156 list_total: 33 fname_total: 116 hit_total: 4633 miss_total: 1576 mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.batch_stats= snapshot_time: 1785396850.981221843 secs.nsecs start_time: 1785396720.054035399 secs.nsecs elapsed_time: 130.927186444 secs.nsecs subreqs per batch batches %% cum %% 1: 2 5 5 2: 2 5 10 4: 0 0 10 8: 4 10 21 16: 22 59 81 32: 7 18 100 mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.batch_stats= snapshot_time: 1785396850.981286223 secs.nsecs start_time: 1785396720.054101271 secs.nsecs elapsed_time: 130.927184952 secs.nsecs subreqs per batch batches %% cum %% 1: 1 2 2 2: 0 0 2 4: 3 7 10 8: 1 2 12 16: 11 28 41 32: 23 58 100 - open/close 402 (time 1785396865.19 total 10.00 last 40.18) - open/close 757 (time 1785396875.21 total 20.02 last 35.44) - open/close 1074 (time 1785396885.24 total 30.05 last 31.60) - open/close 1360 (time 1785396895.27 total 40.09 last 28.50) - open/close 1792 (time 1785396905.28 total 50.09 last 43.19) - open/close 2263 (time 1785396915.28 total 60.09 last 47.09) - open/close 2745 (time 1785396925.28 total 70.09 last 48.20) - open/close 3552 (time 1785396935.32 total 80.14 last 80.37) - open/close 4067 (time 1785396945.34 total 90.15 last 51.43) - open/close 4554 (time 1785396955.35 total 100.17 last 48.62) - open/close 5231 (time 1785396965.36 total 110.17 last 67.66) - open/close 5708 (time 1785396975.36 total 120.18 last 47.68) - open/close 6571 (time 1785396985.36 total 130.18 last 86.29) - open/close 7367 (time 1785396995.36 total 140.18 last 79.60) - open/close 7844 (time 1785397005.38 total 150.19 last 47.64) - open/close 8171 (time 1785397015.39 total 160.20 last 32.67) - open/close 8485 (time 1785397025.40 total 170.21 last 31.37) - open/close 8867 (time 1785397035.43 total 180.25 last 38.07) - open/close 9460 (time 1785397045.44 total 190.26 last 59.22) - open/close 9898 (time 1785397055.46 total 200.27 last 43.74) total: 10000 open/close in 202.53 seconds: 49.37 ops/second llite.lustre-ffff955c8cf8a800.statahead_batch_max=0 llite.lustre-ffff955c8cf8a800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.stats=clear mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m27.767s user 0m0.280s sys 0m15.249s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff955c8cf8a800.statahead_batch_max=32 llite.lustre-ffff955c8cf8a800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.stats=clear mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.stats=clear ls -l files in d123ad.sanity: 10001 real 2m39.964s user 0m0.305s sys 0m14.991s batched statahead RPC= hit count=9978 mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.batch_stats= snapshot_time: 1785397341.879237946 secs.nsecs start_time: 1785397160.820184612 secs.nsecs elapsed_time: 181.059053334 secs.nsecs subreqs per batch batches %% cum %% 1: 1 0 0 2: 0 0 0 4: 0 0 0 8: 1 0 0 16: 1 0 0 32: 313 99 100 mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.batch_stats= snapshot_time: 1785397341.879291767 secs.nsecs start_time: 1785397160.820392658 secs.nsecs elapsed_time: 181.058899109 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.batch_stats=0 llite.lustre-ffff955c8cf8a800.statahead_max=2048 llite.lustre-ffff955c8cf8a800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 3.73 seconds: 26.82 ops/second llite.lustre-ffff955c8cf8a800.statahead_max=0 101 real 0m3.549s user 0m0.006s sys 0m0.344s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff955c8cf8a800.statahead_max=2048 llite.lustre-ffff955c8cf8a800.statahead_max=2048 101 real 0m1.252s user 0m0.004s sys 0m0.203s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10077 miss_total: 23 swrong=0 ewrong=0 - open/close 490 (time 1785397777.10 total 10.01 last 48.94) total: 900 open/close in 18.48 seconds: 48.70 ops/second llite.lustre-ffff955c8cf8a800.statahead_max=0 1001 real 0m34.589s user 0m0.039s sys 0m3.228s 'ls -l' 1000 files without statahead: 35 sec llite.lustre-ffff955c8cf8a800.statahead_max=2048 llite.lustre-ffff955c8cf8a800.statahead_max=2048 1001 real 0m7.245s user 0m0.018s sys 0m1.486s 'ls -l' 1000 files with statahead: 8 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11076 miss_total: 24 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 28 seconds llite.lustre-ffff955c8cf8a800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11076 miss_total: 24 mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.batch_stats= snapshot_time: 1785397872.138997310 secs.nsecs start_time: 1785397345.948132935 secs.nsecs elapsed_time: 526.190864375 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 0 0 0 8: 4 66 66 16: 0 0 66 32: 0 0 66 64: 0 0 66 128: 0 0 66 256: 2 33 100 mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.batch_stats= snapshot_time: 1785397872.139060636 secs.nsecs start_time: 1785397345.948245058 secs.nsecs elapsed_time: 526.190815578 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 22 22 8: 1 11 33 16: 2 22 55 32: 1 11 66 64: 0 0 66 128: 1 11 77 256: 2 22 100 - open/close 574 (time 1785397886.38 total 10.01 last 57.36) - open/close 1196 (time 1785397896.38 total 20.01 last 62.19) - open/close 1744 (time 1785397906.39 total 30.02 last 54.76) - open/close 2267 (time 1785397916.39 total 40.02 last 52.29) - open/close 2706 (time 1785397926.39 total 50.02 last 43.89) - open/close 3360 (time 1785397936.39 total 60.02 last 65.40) - open/close 3963 (time 1785397946.41 total 70.04 last 60.19) - open/close 4525 (time 1785397956.44 total 80.06 last 56.06) - open/close 4998 (time 1785397966.46 total 90.09 last 47.18) - open/close 5433 (time 1785397976.47 total 100.10 last 43.47) - open/close 5861 (time 1785397986.48 total 110.11 last 42.77) - open/close 6342 (time 1785397996.49 total 120.12 last 48.05) - open/close 6938 (time 1785398006.50 total 130.13 last 59.53) - open/close 7497 (time 1785398016.51 total 140.14 last 55.83) - open/close 8023 (time 1785398026.51 total 150.14 last 52.59) - open/close 8702 (time 1785398036.51 total 160.14 last 67.90) - open/close 9751 (time 1785398046.52 total 170.15 last 104.84) total: 10000 open/close in 173.65 seconds: 57.59 ops/second llite.lustre-ffff955c8cf8a800.statahead_batch_max=0 llite.lustre-ffff955c8cf8a800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.stats=clear mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m23.438s user 0m0.299s sys 0m14.854s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff955c8cf8a800.statahead_batch_max=256 llite.lustre-ffff955c8cf8a800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.stats=clear mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m12.227s user 0m0.308s sys 0m14.444s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff955c8cf8a800.batch_stats= snapshot_time: 1785398243.515115031 secs.nsecs start_time: 1785398147.964293475 secs.nsecs elapsed_time: 95.550821556 secs.nsecs subreqs per batch batches %% cum %% 1: 164 44 44 2: 78 21 65 4: 36 9 75 8: 24 6 81 16: 10 2 84 32: 22 5 90 64: 2 0 90 128: 0 0 90 256: 34 9 100 mdc.lustre-MDT0001-mdc-ffff955c8cf8a800.batch_stats= snapshot_time: 1785398243.515188661 secs.nsecs start_time: 1785398147.964408942 secs.nsecs elapsed_time: 95.550779719 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff955c8cf8a800.statahead_batch_max=64 llite.lustre-ffff955c8cf8a800.statahead_max=128