== sanity test 123ad: Verify batching statahead works correctly ========================================================== 05:42:10 (1785404530) batch: statahead_max=32 statahead_batch_max=32 mdc.lustre-MDT0000-mdc-ffff963d8806b800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff963d8806b800.batch_stats=0 llite.lustre-ffff963d8806b800.statahead_max=32 llite.lustre-ffff963d8806b800.statahead_batch_max=32 seq.cli-lustre-OST0000-super.width=0x1ffffff seq.cli-lustre-OST0001-super.width=0x1ffffff kvm mdc.lustre-MDT0000-mdc-ffff963d8806b800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff963d8806b800.batch_stats=0 striped dir -i1 -c2 -H crush /mnt/lustre/d123ad.sanity total: 100 open/close in 4.37 seconds: 22.87 ops/second llite.lustre-ffff963d8806b800.statahead_max=0 101 real 0m3.414s user 0m0.006s sys 0m0.333s 'ls -l' 100 files without statahead: 3 sec llite.lustre-ffff963d8806b800.statahead_max=32 llite.lustre-ffff963d8806b800.statahead_max=32 101 real 0m2.014s user 0m0.003s sys 0m0.194s 'ls -l' 100 files with statahead: 2 sec statahead total: 164 statahead wrong: 2 agl total: 156 list_total: 30 fname_total: 123 hit_total: 4084 miss_total: 1609 swrong=2 ewrong=2 - open/close 602 (time 1785404569.08 total 10.01 last 60.14) total: 900 open/close in 15.02 seconds: 59.93 ops/second llite.lustre-ffff963d8806b800.statahead_max=0 1001 real 0m42.599s user 0m0.030s sys 0m3.169s 'ls -l' 1000 files without statahead: 43 sec llite.lustre-ffff963d8806b800.statahead_max=32 llite.lustre-ffff963d8806b800.statahead_max=32 1001 real 0m10.148s user 0m0.018s sys 0m1.440s 'ls -l' 1000 files with statahead: 10 sec statahead total: 165 statahead wrong: 2 agl total: 157 list_total: 31 fname_total: 123 hit_total: 5083 miss_total: 1610 swrong=2 ewrong=2 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 30 seconds llite.lustre-ffff963d8806b800.statahead_stats= statahead total: 165 statahead wrong: 2 agl total: 157 list_total: 31 fname_total: 123 hit_total: 5083 miss_total: 1610 mdc.lustre-MDT0000-mdc-ffff963d8806b800.batch_stats= snapshot_time: 1785404679.399443565 secs.nsecs start_time: 1785404536.915822415 secs.nsecs elapsed_time: 142.483621150 secs.nsecs subreqs per batch batches %% cum %% 1: 3 8 8 2: 0 0 8 4: 1 2 10 8: 4 10 21 16: 25 67 89 32: 4 10 100 mdc.lustre-MDT0001-mdc-ffff963d8806b800.batch_stats= snapshot_time: 1785404679.399524260 secs.nsecs start_time: 1785404536.916735440 secs.nsecs elapsed_time: 142.482788820 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 1 2 2 4: 2 5 7 8: 1 2 10 16: 8 20 30 32: 27 69 100 - open/close 607 (time 1785404692.98 total 10.07 last 60.25) - open/close 1134 (time 1785404702.98 total 20.08 last 52.69) - open/close 1669 (time 1785404712.99 total 30.08 last 53.48) - open/close 2150 (time 1785404723.00 total 40.09 last 48.05) - open/close 2716 (time 1785404733.00 total 50.10 last 56.57) - open/close 3278 (time 1785404743.01 total 60.10 last 56.17) - open/close 3826 (time 1785404753.02 total 70.11 last 54.74) - open/close 4369 (time 1785404763.05 total 80.14 last 54.16) - open/close 4879 (time 1785404773.06 total 90.15 last 50.93) - open/close 5444 (time 1785404783.07 total 100.16 last 56.48) - open/close 6402 (time 1785404793.07 total 110.16 last 95.74) - open/close 7281 (time 1785404803.07 total 120.16 last 87.89) - open/close 7827 (time 1785404813.08 total 130.17 last 54.58) - open/close 8455 (time 1785404823.08 total 140.17 last 62.79) - open/close 9238 (time 1785404833.09 total 150.18 last 78.20) - open/close 9898 (time 1785404843.13 total 160.22 last 65.76) total: 10000 open/close in 161.14 seconds: 62.06 ops/second llite.lustre-ffff963d8806b800.statahead_batch_max=0 llite.lustre-ffff963d8806b800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff963d8806b800.stats=clear mdc.lustre-MDT0001-mdc-ffff963d8806b800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m28.498s user 0m0.298s sys 0m14.668s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff963d8806b800.statahead_batch_max=32 llite.lustre-ffff963d8806b800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff963d8806b800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff963d8806b800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff963d8806b800.stats=clear mdc.lustre-MDT0001-mdc-ffff963d8806b800.stats=clear ls -l files in d123ad.sanity: 10001 real 2m31.109s user 0m0.277s sys 0m14.619s batched statahead RPC= hit count=9971 mdc.lustre-MDT0000-mdc-ffff963d8806b800.batch_stats= snapshot_time: 1785405122.504214883 secs.nsecs start_time: 1785404944.996265575 secs.nsecs elapsed_time: 177.507949308 secs.nsecs subreqs per batch batches %% cum %% 1: 1 0 0 2: 1 0 0 4: 1 0 0 8: 1 0 1 16: 1 0 1 32: 313 98 100 mdc.lustre-MDT0001-mdc-ffff963d8806b800.batch_stats= snapshot_time: 1785405122.504474483 secs.nsecs start_time: 1785404944.996330120 secs.nsecs elapsed_time: 177.508144363 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 batch: statahead_max=2048 statahead_batch_max=256 mdc.lustre-MDT0000-mdc-ffff963d8806b800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff963d8806b800.batch_stats=0 llite.lustre-ffff963d8806b800.statahead_max=2048 llite.lustre-ffff963d8806b800.statahead_batch_max=256 kvm mdc.lustre-MDT0000-mdc-ffff963d8806b800.batch_stats=0 mdc.lustre-MDT0001-mdc-ffff963d8806b800.batch_stats=0 striped dir -i1 -c2 -H fnv_1a_64 /mnt/lustre/d123ad.sanity total: 100 open/close in 3.77 seconds: 26.56 ops/second llite.lustre-ffff963d8806b800.statahead_max=0 101 real 0m3.875s user 0m0.004s sys 0m0.324s 'ls -l' 100 files without statahead: 4 sec llite.lustre-ffff963d8806b800.statahead_max=2048 llite.lustre-ffff963d8806b800.statahead_max=2048 101 real 0m1.123s user 0m0.007s sys 0m0.191s 'ls -l' 100 files with statahead: 1 sec statahead total: 2 statahead wrong: 0 agl total: 2 list_total: 2 fname_total: 0 hit_total: 10070 miss_total: 30 swrong=0 ewrong=0 - open/close 638 (time 1785405568.90 total 10.01 last 63.74) total: 900 open/close in 14.91 seconds: 60.36 ops/second llite.lustre-ffff963d8806b800.statahead_max=0 1001 real 0m39.975s user 0m0.036s sys 0m3.290s 'ls -l' 1000 files without statahead: 40 sec llite.lustre-ffff963d8806b800.statahead_max=2048 llite.lustre-ffff963d8806b800.statahead_max=2048 1001 real 0m11.098s user 0m0.026s sys 0m1.411s 'ls -l' 1000 files with statahead: 11 sec statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11069 miss_total: 31 swrong=0 ewrong=0 'ls -l' done unlink + rm -r /mnt/lustre/d123ad.sanity: 24 seconds llite.lustre-ffff963d8806b800.statahead_stats= statahead total: 3 statahead wrong: 0 agl total: 3 list_total: 3 fname_total: 0 hit_total: 11069 miss_total: 31 mdc.lustre-MDT0000-mdc-ffff963d8806b800.batch_stats= snapshot_time: 1785405672.545079223 secs.nsecs start_time: 1785405126.496273273 secs.nsecs elapsed_time: 546.048805950 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 1 11 11 4: 0 0 11 8: 4 44 55 16: 1 11 66 32: 0 0 66 64: 1 11 77 128: 0 0 77 256: 2 22 100 mdc.lustre-MDT0001-mdc-ffff963d8806b800.batch_stats= snapshot_time: 1785405672.547463728 secs.nsecs start_time: 1785405126.496523361 secs.nsecs elapsed_time: 546.050940367 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 2: 0 0 0 4: 2 25 25 8: 2 25 50 16: 1 12 62 32: 1 12 75 64: 0 0 75 128: 0 0 75 256: 2 25 100 - open/close 594 (time 1785405685.83 total 10.01 last 59.35) - open/close 1475 (time 1785405695.85 total 20.02 last 87.97) - open/close 2009 (time 1785405705.87 total 30.04 last 53.29) - open/close 2603 (time 1785405715.92 total 40.10 last 59.07) - open/close 3127 (time 1785405725.92 total 50.10 last 52.40) - open/close 3731 (time 1785405735.93 total 60.11 last 60.37) - open/close 4644 (time 1785405745.94 total 70.11 last 91.24) - open/close 5314 (time 1785405755.94 total 80.11 last 66.98) - open/close 5923 (time 1785405765.94 total 90.12 last 60.89) - open/close 6471 (time 1785405775.96 total 100.14 last 54.68) - open/close 6908 (time 1785405785.97 total 110.14 last 43.68) - open/close 7317 (time 1785405795.97 total 120.14 last 40.90) - open/close 7756 (time 1785405805.99 total 130.17 last 43.80) - open/close 8397 (time 1785405816.00 total 140.17 last 64.05) - open/close 9072 (time 1785405826.00 total 150.18 last 67.47) - open/close 9671 (time 1785405836.00 total 160.18 last 59.90) total: 10000 open/close in 166.38 seconds: 60.10 ops/second llite.lustre-ffff963d8806b800.statahead_batch_max=0 llite.lustre-ffff963d8806b800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff963d8806b800.stats=clear mdc.lustre-MDT0001-mdc-ffff963d8806b800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m52.820s user 0m0.300s sys 0m15.652s unbatched statahead RPC=10004 hit count=9999 llite.lustre-ffff963d8806b800.statahead_batch_max=256 llite.lustre-ffff963d8806b800.statahead_stats=clear mdc.lustre-MDT0000-mdc-ffff963d8806b800.batch_stats=clear mdc.lustre-MDT0001-mdc-ffff963d8806b800.batch_stats=clear mdc.lustre-MDT0000-mdc-ffff963d8806b800.stats=clear mdc.lustre-MDT0001-mdc-ffff963d8806b800.stats=clear ls -l files in d123ad.sanity: 10001 real 1m16.137s user 0m0.288s sys 0m14.212s batched statahead RPC= hit count=9999 mdc.lustre-MDT0000-mdc-ffff963d8806b800.batch_stats= snapshot_time: 1785406072.680783947 secs.nsecs start_time: 1785405970.209296724 secs.nsecs elapsed_time: 102.471487223 secs.nsecs subreqs per batch batches %% cum %% 1: 195 55 55 2: 47 13 68 4: 32 9 77 8: 14 3 81 16: 7 1 83 32: 17 4 88 64: 8 2 90 128: 0 0 90 256: 34 9 100 mdc.lustre-MDT0001-mdc-ffff963d8806b800.batch_stats= snapshot_time: 1785406072.680852441 secs.nsecs start_time: 1785405970.209384133 secs.nsecs elapsed_time: 102.471468308 secs.nsecs subreqs per batch batches %% cum %% 1: 0 0 0 unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second unlink(/mnt/lustre/d123ad.sanity/f123ad.sanity0) error: No such file or directory total: 0 unlinks in 0 seconds: -nan unlinks/second seq.cli-lustre-OST0000-super.width=65536 seq.cli-lustre-OST0001-super.width=65536 llite.lustre-ffff963d8806b800.statahead_batch_max=64 llite.lustre-ffff963d8806b800.statahead_max=128