== sanity test 182b: Test parallel modify metadata operations from osp ========================================================== 02:55:25 (1788936925)
osp.lustre-MDT0000-osp-MDT0001.rpc_stats
osp.lustre-MDT0001-osp-MDT0000.rpc_stats
total: 10 mkdir in 0.78 seconds: 12.82 ops/second
 - mkdir 127 (time 1788936951.65 total 10.04 last 12.64)
 - mkdir 233 (time 1788936961.67 total 20.07 last 10.58)
 - mkdir 351 (time 1788936971.72 total 30.11 last 11.74)
 - mkdir 471 (time 1788936981.80 total 40.19 last 11.91)
 - mkdir 574 (time 1788936991.84 total 50.23 last 10.26)
 - mkdir 705 (time 1788937001.88 total 60.27 last 13.05)
 - mkdir 811 (time 1788937011.92 total 70.32 last 10.55)
 - mkdir 907 (time 1788937021.94 total 80.33 last 9.58)
 - mkdir 980 (time 1788937031.97 total 90.36 last 7.28)
total: 1000 mkdir in 92.64 seconds: 10.79 ops/second
 - mkdir 127 (time 1788936951.18 total 10.03 last 12.66)
 - mkdir 226 (time 1788936961.22 total 20.07 last 9.86)
 - mkdir 341 (time 1788936971.25 total 30.10 last 11.47)
 - mkdir 458 (time 1788936981.27 total 40.12 last 11.68)
 - mkdir 561 (time 1788936991.33 total 50.18 last 10.24)
 - mkdir 687 (time 1788937001.39 total 60.24 last 12.53)
 - mkdir 791 (time 1788937011.44 total 70.29 last 10.34)
 - mkdir 889 (time 1788937021.48 total 80.33 last 9.76)
 - mkdir 963 (time 1788937031.53 total 90.38 last 7.37)
total: 1000 mkdir in 95.01 seconds: 10.53 ops/second
 - mkdir 115 (time 1788936949.77 total 10.04 last 11.45)
 - mkdir 227 (time 1788936959.88 total 20.14 last 11.09)
 - mkdir 337 (time 1788936969.98 total 30.25 last 10.89)
 - mkdir 449 (time 1788936980.09 total 40.36 last 11.08)
 - mkdir 550 (time 1788936990.12 total 50.38 last 10.07)
 - mkdir 668 (time 1788937000.17 total 60.43 last 11.74)
 - mkdir 772 (time 1788937010.19 total 70.46 last 10.37)
 - mkdir 877 (time 1788937020.24 total 80.51 last 10.45)
 - mkdir 951 (time 1788937030.36 total 90.63 last 7.31)
total: 1000 mkdir in 96.47 seconds: 10.37 ops/second
 - mkdir 131 (time 1788936951.21 total 10.09 last 12.98)
 - mkdir 231 (time 1788936961.28 total 20.16 last 9.93)
 - mkdir 343 (time 1788936971.31 total 30.19 last 11.17)
 - mkdir 462 (time 1788936981.34 total 40.22 last 11.86)
 - mkdir 560 (time 1788936991.39 total 50.27 last 9.76)
 - mkdir 684 (time 1788937001.42 total 60.30 last 12.36)
 - mkdir 789 (time 1788937011.47 total 70.35 last 10.44)
 - mkdir 888 (time 1788937021.49 total 80.37 last 9.88)
 - mkdir 961 (time 1788937031.55 total 90.43 last 7.26)
total: 1000 mkdir in 95.35 seconds: 10.49 ops/second
 - mkdir 126 (time 1788936951.41 total 10.03 last 12.56)
 - mkdir 228 (time 1788936961.50 total 20.12 last 10.11)
 - mkdir 343 (time 1788936971.61 total 30.24 last 11.36)
 - mkdir 459 (time 1788936981.64 total 40.26 last 11.57)
 - mkdir 558 (time 1788936991.72 total 50.34 last 9.82)
 - mkdir 681 (time 1788937001.76 total 60.38 last 12.26)
 - mkdir 788 (time 1788937011.83 total 70.45 last 10.62)
 - mkdir 889 (time 1788937021.84 total 80.46 last 10.09)
 - mkdir 963 (time 1788937032.04 total 90.66 last 7.25)
total: 1000 mkdir in 95.13 seconds: 10.51 ops/second
 - mkdir 125 (time 1788936951.35 total 10.01 last 12.49)
 - mkdir 225 (time 1788936961.45 total 20.11 last 9.90)
 - mkdir 338 (time 1788936971.50 total 30.16 last 11.24)
 - mkdir 459 (time 1788936981.54 total 40.20 last 12.06)
 - mkdir 560 (time 1788936991.54 total 50.20 last 10.10)
 - mkdir 682 (time 1788937001.62 total 60.28 last 12.10)
 - mkdir 785 (time 1788937011.68 total 70.34 last 10.24)
 - mkdir 883 (time 1788937021.69 total 80.35 last 9.79)
 - mkdir 957 (time 1788937031.72 total 90.38 last 7.38)
total: 1000 mkdir in 96.01 seconds: 10.42 ops/second
 - mkdir 128 (time 1788936951.42 total 10.02 last 12.78)
 - mkdir 228 (time 1788936961.51 total 20.11 last 9.91)
 - mkdir 339 (time 1788936971.66 total 30.27 last 10.93)
 - mkdir 457 (time 1788936981.71 total 40.32 last 11.74)
 - mkdir 555 (time 1788936991.82 total 50.42 last 9.70)
 - mkdir 677 (time 1788937001.82 total 60.42 last 12.20)
 - mkdir 778 (time 1788937011.84 total 70.45 last 10.08)
 - mkdir 880 (time 1788937021.87 total 80.47 last 10.18)
 - mkdir 955 (time 1788937031.97 total 90.57 last 7.42)
total: 1000 mkdir in 96.46 seconds: 10.37 ops/second
 - mkdir 114 (time 1788936949.60 total 10.03 last 11.37)
 - mkdir 218 (time 1788936959.75 total 20.18 last 10.25)
 - mkdir 325 (time 1788936969.79 total 30.22 last 10.65)
 - mkdir 435 (time 1788936979.87 total 40.30 last 10.91)
 - mkdir 535 (time 1788936989.96 total 50.38 last 9.92)
 - mkdir 648 (time 1788937000.01 total 60.44 last 11.24)
 - mkdir 747 (time 1788937010.05 total 70.48 last 9.86)
 - mkdir 849 (time 1788937020.05 total 80.48 last 10.20)
 - mkdir 924 (time 1788937030.21 total 90.63 last 7.39)
 - mkdir 998 (time 1788937040.25 total 100.68 last 7.37)
total: 1000 mkdir in 100.81 seconds: 9.92 ops/second
 - mkdir 118 (time 1788936951.51 total 10.01 last 11.79)
 - mkdir 210 (time 1788936961.57 total 20.07 last 9.14)
 - mkdir 323 (time 1788936971.65 total 30.15 last 11.21)
 - mkdir 441 (time 1788936981.71 total 40.21 last 11.73)
 - mkdir 538 (time 1788936991.81 total 50.31 last 9.61)
 - mkdir 661 (time 1788937001.86 total 60.35 last 12.24)
 - mkdir 760 (time 1788937011.90 total 70.40 last 9.85)
 - mkdir 857 (time 1788937021.96 total 80.46 last 9.64)
 - mkdir 930 (time 1788937031.97 total 90.47 last 7.29)
 - mkdir 999 (time 1788937042.09 total 100.59 last 6.82)
total: 1000 mkdir in 100.59 seconds: 9.94 ops/second
 - mkdir 115 (time 1788936951.04 total 10.04 last 11.46)
 - mkdir 211 (time 1788936961.13 total 20.13 last 9.51)
 - mkdir 314 (time 1788936971.24 total 30.24 last 10.19)
 - mkdir 423 (time 1788936981.30 total 40.30 last 10.83)
 - mkdir 518 (time 1788936991.30 total 50.30 last 9.50)
 - mkdir 631 (time 1788937001.32 total 60.32 last 11.28)
 - mkdir 727 (time 1788937011.33 total 70.34 last 9.58)
 - mkdir 827 (time 1788937021.37 total 80.38 last 9.96)
 - mkdir 902 (time 1788937031.43 total 90.43 last 7.46)
 - mkdir 976 (time 1788937041.56 total 100.56 last 7.31)
total: 1000 mkdir in 103.05 seconds: 9.70 ops/second
Time for file creation 115 sec for 8 parallel RPCs
rmdir(/mnt/lustre/d182b.sanity/7/d-0) error: No such file or directory
total: 0 unlinks in 0 seconds: -nan unlinks/second
rmdir(/mnt/lustre/d182b.sanity/3/d-0) error: No such file or directory
total: 0 unlinks in 0 seconds: -nan unlinks/second
rmdir(/mnt/lustre/d182b.sanity/8/d-0) error: No such file or directory
total: 0 unlinks in 0 seconds: -nan unlinks/second
rmdir(/mnt/lustre/d182b.sanity/0/d-0) error: No such file or directory
total: 0 unlinks in 0 seconds: -nan unlinks/second
rmdir(/mnt/lustre/d182b.sanity/9/d-0) error: No such file or directory
total: 0 unlinks in 0 seconds: -nan unlinks/second
rmdir(/mnt/lustre/d182b.sanity/5/d-0) error: No such file or directory
total: 0 unlinks in 0 seconds: -nan unlinks/second
rmdir(/mnt/lustre/d182b.sanity/1/d-0) error: No such file or directory
total: 0 unlinks in 0 seconds: -nan unlinks/second
rmdir(/mnt/lustre/d182b.sanity/4/d-0) error: No such file or directory
total: 0 unlinks in 0 seconds: -nan unlinks/second
rmdir(/mnt/lustre/d182b.sanity/6/d-0) error: No such file or directory
total: 0 unlinks in 0 seconds: -nan unlinks/second
rmdir(/mnt/lustre/d182b.sanity/2/d-0) error: No such file or directory
total: 0 unlinks in 0 seconds: -nan unlinks/second
Time for file removal 17 sec for 8 parallel RPCs
osp.lustre-MDT0001-osp-MDT0000.max_mod_rpcs_in_flight=1
total: 10 mkdir in 0.86 seconds: 11.58 ops/second
 - mkdir 110 (time 1788937759.45 total 10.05 last 10.95)
 - mkdir 215 (time 1788937769.48 total 20.08 last 10.47)
 - mkdir 345 (time 1788937779.49 total 30.09 last 12.99)
 - mkdir 468 (time 1788937789.54 total 40.14 last 12.24)
 - mkdir 600 (time 1788937799.59 total 50.18 last 13.14)
 - mkdir 747 (time 1788937809.64 total 60.24 last 14.62)
 - mkdir 877 (time 1788937819.68 total 70.28 last 12.95)
total: 1000 mkdir in 79.87 seconds: 12.52 ops/second
 - mkdir 125 (time 1788937761.79 total 10.00 last 12.49)
 - mkdir 233 (time 1788937771.86 total 20.08 last 10.72)
 - mkdir 374 (time 1788937781.87 total 30.09 last 14.09)
 - mkdir 497 (time 1788937791.90 total 40.12 last 12.26)
 - mkdir 627 (time 1788937801.92 total 50.13 last 12.98)
 - mkdir 776 (time 1788937811.97 total 60.18 last 14.83)
 - mkdir 897 (time 1788937821.97 total 70.19 last 12.09)
total: 1000 mkdir in 79.04 seconds: 12.65 ops/second
 - mkdir 118 (time 1788937764.20 total 10.06 last 11.73)
 - mkdir 241 (time 1788937774.23 total 20.09 last 12.26)
 - mkdir 383 (time 1788937784.27 total 30.13 last 14.14)
 - mkdir 509 (time 1788937794.30 total 40.16 last 12.57)
 - mkdir 641 (time 1788937804.30 total 50.16 last 13.19)
 - mkdir 785 (time 1788937814.38 total 60.24 last 14.30)
 - mkdir 918 (time 1788937824.40 total 70.26 last 13.27)
total: 1000 mkdir in 77.99 seconds: 12.82 ops/second
 - mkdir 124 (time 1788937762.76 total 10.12 last 12.25)
 - mkdir 233 (time 1788937772.80 total 20.16 last 10.86)
 - mkdir 371 (time 1788937782.89 total 30.25 last 13.67)
 - mkdir 498 (time 1788937792.90 total 40.27 last 12.68)
 - mkdir 631 (time 1788937802.91 total 50.27 last 13.29)
 - mkdir 782 (time 1788937812.91 total 60.28 last 15.10)
 - mkdir 897 (time 1788937822.97 total 70.33 last 11.44)
 - mkdir 998 (time 1788937833.00 total 80.36 last 10.07)
total: 1000 mkdir in 80.55 seconds: 12.41 ops/second
 - mkdir 121 (time 1788937763.82 total 10.01 last 12.08)
 - mkdir 231 (time 1788937773.85 total 20.04 last 10.97)
 - mkdir 368 (time 1788937783.87 total 30.06 last 13.68)
 - mkdir 496 (time 1788937793.90 total 40.09 last 12.75)
 - mkdir 632 (time 1788937803.93 total 50.12 last 13.56)
 - mkdir 778 (time 1788937814.04 total 60.23 last 14.45)
 - mkdir 905 (time 1788937824.11 total 70.30 last 12.61)
 - mkdir 996 (time 1788937834.11 total 80.30 last 9.10)
total: 1000 mkdir in 81.20 seconds: 12.32 ops/second
 - mkdir 120 (time 1788937764.02 total 10.02 last 11.97)
 - mkdir 228 (time 1788937774.12 total 20.13 last 10.69)
 - mkdir 369 (time 1788937784.15 total 30.16 last 14.06)
 - mkdir 495 (time 1788937794.26 total 40.26 last 12.47)
 - mkdir 631 (time 1788937804.28 total 50.28 last 13.57)
 - mkdir 778 (time 1788937814.30 total 60.30 last 14.67)
 - mkdir 902 (time 1788937824.30 total 70.31 last 12.39)
 - mkdir 994 (time 1788937834.41 total 80.41 last 9.10)
total: 1000 mkdir in 81.98 seconds: 12.20 ops/second
 - mkdir 118 (time 1788937764.00 total 10.09 last 11.69)
 - mkdir 229 (time 1788937774.01 total 20.10 last 11.09)
 - mkdir 365 (time 1788937784.08 total 30.18 last 13.50)
 - mkdir 490 (time 1788937794.17 total 40.26 last 12.40)
 - mkdir 622 (time 1788937804.19 total 50.29 last 13.16)
 - mkdir 766 (time 1788937814.24 total 60.34 last 14.33)
 - mkdir 890 (time 1788937824.29 total 70.38 last 12.34)
 - mkdir 979 (time 1788937834.46 total 80.55 last 8.75)
total: 1000 mkdir in 85.63 seconds: 11.68 ops/second
 - mkdir 115 (time 1788937763.21 total 10.10 last 11.39)
 - mkdir 222 (time 1788937773.25 total 20.14 last 10.66)
 - mkdir 357 (time 1788937783.31 total 30.20 last 13.41)
 - mkdir 485 (time 1788937793.35 total 40.24 last 12.76)
 - mkdir 616 (time 1788937803.38 total 50.27 last 13.06)
 - mkdir 760 (time 1788937813.44 total 60.33 last 14.31)
 - mkdir 876 (time 1788937823.50 total 70.39 last 11.53)
 - mkdir 972 (time 1788937833.77 total 80.67 last 9.35)
total: 1000 mkdir in 87.20 seconds: 11.47 ops/second
 - mkdir 121 (time 1788937764.57 total 10.11 last 11.97)
 - mkdir 231 (time 1788937774.65 total 20.18 last 10.91)
 - mkdir 363 (time 1788937784.73 total 30.26 last 13.10)
 - mkdir 489 (time 1788937794.77 total 40.31 last 12.55)
 - mkdir 624 (time 1788937804.81 total 50.34 last 13.45)
 - mkdir 766 (time 1788937814.86 total 60.39 last 14.13)
 - mkdir 890 (time 1788937824.87 total 70.40 last 12.39)
 - mkdir 972 (time 1788937835.01 total 80.55 last 8.08)
total: 1000 mkdir in 86.62 seconds: 11.54 ops/second
 - mkdir 114 (time 1788937764.46 total 10.04 last 11.36)
 - mkdir 218 (time 1788937774.50 total 20.08 last 10.36)
 - mkdir 348 (time 1788937784.52 total 30.10 last 12.97)
 - mkdir 474 (time 1788937794.55 total 40.12 last 12.57)
 - mkdir 609 (time 1788937804.59 total 50.17 last 13.44)
 - mkdir 754 (time 1788937814.66 total 60.23 last 14.41)
 - mkdir 872 (time 1788937824.70 total 70.27 last 11.75)
 - mkdir 954 (time 1788937834.79 total 80.37 last 8.12)
total: 1000 mkdir in 89.00 seconds: 11.24 ops/second
Time for file creation 104 sec for 1 RPC sent at a time
 - unlinked 0 (time 1788937855 ; total 0 ; last 0)
total: 1000 unlinks in 40 seconds: 25.000000 unlinks/second
 - unlinked 0 (time 1788937856 ; total 0 ; last 0)
total: 1000 unlinks in 42 seconds: 23.809525 unlinks/second
 - unlinked 0 (time 1788937857 ; total 0 ; last 0)
total: 1000 unlinks in 41 seconds: 24.390244 unlinks/second
 - unlinked 0 (time 1788937858 ; total 0 ; last 0)
total: 1000 unlinks in 40 seconds: 25.000000 unlinks/second
 - unlinked 0 (time 1788937857 ; total 0 ; last 0)
total: 1000 unlinks in 42 seconds: 23.809525 unlinks/second
 - unlinked 0 (time 1788937857 ; total 0 ; last 0)
total: 1000 unlinks in 42 seconds: 23.809525 unlinks/second
 - unlinked 0 (time 1788937858 ; total 1 ; last 1)
total: 1000 unlinks in 42 seconds: 23.809525 unlinks/second
 - unlinked 0 (time 1788937858 ; total 0 ; last 0)
total: 1000 unlinks in 41 seconds: 24.390244 unlinks/second
 - unlinked 0 (time 1788937858 ; total 0 ; last 0)
total: 1000 unlinks in 41 seconds: 24.390244 unlinks/second
 - unlinked 0 (time 1788937858 ; total 0 ; last 0)
total: 1000 unlinks in 41 seconds: 24.390244 unlinks/second
Time for file removal 57 sec for 1 RPC sent at a time
osp.lustre-MDT0001-osp-MDT0000.max_mod_rpcs_in_flight=8